From 105cd11bf9335331d4b7bb77249b92a5a24f74cc Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 14:17:59 +0800
Subject: [PATCH 01/66] test: modify checkpackages scritps

---
 packaging/testpackage.sh | 104 ++++++++++++++++++++++-----------------
 1 file changed, 59 insertions(+), 45 deletions(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index 56da9e59be..b0edf2fc19 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -68,11 +68,36 @@ fi
 }
 
 
+function wgetFile {
+
+file=$1
+
+if [ ! -f  ${file}  ];then
+    echoColor  BD "wget https://www.taosdata.com/assets-download/3.0/${file}"
+    wget https://www.taosdata.com/assets-download/3.0/${file}
+else
+    echoColor  YD "${file} already exists "
+fi
+}
+
+function newPath {
+
+buildPath=$1
+
+if [ ! -d ${buildPath} ] ;then
+    echoColor BD "mkdir -p ${buildPath}"
+    mkdir -p ${buildPath}
+else  
+    echoColor YD "${buildPath} already exists"
+fi
+
+}
+
+
 echoColor G "===== install basesoft ====="
 
 cmdInstall tree
 cmdInstall wget
-cmdInstall sshpass
 
 echoColor G "===== Uninstall all components of TDeingne ====="
 
@@ -97,11 +122,14 @@ echoColor G "===== new workroom path ====="
 installPath="/usr/local/src/packageTest"
 oriInstallPath="/usr/local/src/packageTest/3.1"
 
-if [ ! -d ${installPath} ] ;then
-    echoColor BD "mkdir -p ${installPath}"
-    mkdir -p ${installPath}
-else  
-    echoColor YD "${installPath} already exists"
+newPath ${installPath}
+
+newPath ${oriInstallPath}
+
+
+if [ -d ${oriInstallPath}/${originTdpPath} ] ;then
+    echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*"
+    rm -rf  ${oriInstallPath}/${originTdpPath}/*  
 fi
 
 if [ -d ${installPath}/${tdPath} ] ;then
@@ -109,33 +137,13 @@ if [ -d ${installPath}/${tdPath} ] ;then
     rm -rf ${installPath}/${tdPath}/*
 fi
 
-if [ ! -d ${oriInstallPath} ] ;then
-    echoColor BD "mkdir -p ${oriInstallPath}"
-    mkdir -p ${oriInstallPath}
-else  
-    echoColor YD "${oriInstallPath} already exists"
-fi
-
-if [ -d ${oriInstallPath}/${originTdpPath} ] ;then
-    echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*"
-    rm -rf  ${oriInstallPath}/${originTdpPath}/*  
-fi
-
-
 echoColor G "===== download  installPackage ====="
-# cd ${installPath}
-# wget https://www.taosdata.com/assets-download/3.0/${packgeName}
-# cd  ${oriInstallPath}
-# wget https://www.taosdata.com/assets-download/3.0/${originPackageName}
+cd ${installPath} && wgetFile ${packgeName}
+cd  ${oriInstallPath}  && wgetFile ${originPackageName}
 
 cd ${installPath}
 cp -r ${scriptDir}/debRpmAutoInstall.sh   . 
 
-if [ ! -f  {packgeName}  ];then
-    echoColor  BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName}  ."
-    sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName}  .
-fi
-
 packageSuffix=$(echo ${packgeName}  | awk -F '.' '{print $NF}')
 
 
@@ -181,8 +189,7 @@ elif [[ ${packgeName} =~ "tar" ]];then
     cd  ${oriInstallPath}
     if [ ! -f  {originPackageName}  ];then
         echoColor YD "download  base installPackage"
-        echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} ."
-        sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} .
+        wgetFile ${originPackageName}
     fi
     echoColor YD "unzip the base installation package" 
     echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} 
@@ -222,24 +229,31 @@ fi
 
 cd ${installPath}
 
-if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
-    sshpass -p ${password}   scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
-    # wget https://www.taosdata.com/assets-download/3.0/taosTools-2.1.2-Linux-x64.tar.gz
-    tar xf taosTools-2.1.2-Linux-x64.tar.gz
-    cd taosTools-2.1.2 && bash install-taostools.sh
-elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "deb" ]] ;then
-    echoColor G "===== install taos-tools when package is lite or client ====="
-    cd ${installPath}
-    sshpass -p ${password}   scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
-    tar xf taosTools-2.1.2-Linux-x64.tar.gz
-    cd taosTools-2.1.2 && bash install-taostools.sh
-elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "rpm" ]]  ;then
-    echoColor G "===== install taos-tools when package is lite or client ====="
-    cd ${installPath}
-    sshpass -p ${password}   scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
+    wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
     tar xf taosTools-2.1.2-Linux-x64.tar.gz
     cd taosTools-2.1.2 && bash install-taostools.sh
 fi
 
+# if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
+#     echoColor G "===== install taos-tools when package is lite or client ====="
+#     cd ${installPath}
+#     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
+#     tar xf taosTools-2.1.2-Linux-x64.tar.gz
+#     cd taosTools-2.1.2 && bash install-taostools.sh
+# elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "deb" ]] ;then
+#     echoColor G "===== install taos-tools when package is lite or client ====="
+#     cd ${installPath}
+#     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
+#     tar xf taosTools-2.1.2-Linux-x64.tar.gz
+#     cd taosTools-2.1.2 && bash install-taostools.sh
+# elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "rpm" ]]  ;then
+#     echoColor G "===== install taos-tools when package is lite or client ====="
+#     cd ${installPath}
+#     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
+#     tar xf taosTools-2.1.2-Linux-x64.tar.gz
+#     cd taosTools-2.1.2 && bash install-taostools.sh
+# fi
+

From e258ba70816244773efc9fb05745fab1fcd7bfc5 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 14:34:58 +0800
Subject: [PATCH 02/66] test: modify checkpackages scritps

---
 packaging/testpackage.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index b0edf2fc19..e4dbd6a356 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -98,6 +98,7 @@ echoColor G "===== install basesoft ====="
 
 cmdInstall tree
 cmdInstall wget
+cmdInstall expect
 
 echoColor G "===== Uninstall all components of TDeingne ====="
 

From 34efb6d6b706173263e71df8db06e670ab9a00a3 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 1 Sep 2022 15:01:19 +0800
Subject: [PATCH 03/66] test: reopen case

---
 tests/system-test/fulltest.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 4305ceff56..138633868a 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -333,7 +333,7 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
 python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
 python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
 python3 ./test.py -f 7-tmq/tmq_taosx.py
-# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
+python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
 
 #------------querPolicy  2-----------
 

From 11b7f4df31d5c3df8a22b1d8becbb4e2c6f1a6a3 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 15:49:45 +0800
Subject: [PATCH 04/66] test: modify checkpackages scritps

---
 packaging/testpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index e4dbd6a356..6527f9d2c5 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -230,7 +230,7 @@ fi
 
 cd ${installPath}
 
-if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [ ${packgeName} =~ "server" ]])  ([[ ${packgeName} =~ "rpm" ]] && [ ${packgeName} =~ "server" ]]) ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .

From 7dde782247cfb156a76e28a8fc2a161188c3849a Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 16:20:39 +0800
Subject: [PATCH 05/66] test: modify checkpackages scritps

---
 packaging/testpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index 6527f9d2c5..c26b2d5e21 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -230,7 +230,7 @@ fi
 
 cd ${installPath}
 
-if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [ ${packgeName} =~ "server" ]])  ([[ ${packgeName} =~ "rpm" ]] && [ ${packgeName} =~ "server" ]]) ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [ ${packgeName} =~ "server" ]]) ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .

From ea0d429c3e09bf288704c18b9fdc52ed9032f026 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 16:31:21 +0800
Subject: [PATCH 06/66] test: modify checkpackages scritps of  python-connector

---
 packaging/checkPackageRuning.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/checkPackageRuning.py b/packaging/checkPackageRuning.py
index c0d1e8b86c..39839452b6 100755
--- a/packaging/checkPackageRuning.py
+++ b/packaging/checkPackageRuning.py
@@ -24,7 +24,7 @@ import time
 out = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ")
 print("taospy version %s "%out)
 if (out == "" ):
-    os.system("pip install git+https://github.com/taosdata/taos-connector-python.git")
+    os.system("pip3 install git+https://github.com/taosdata/taos-connector-python.git")
     print("install taos python connector")
 else:
     os.system("pip3 install --upgrade  taospy  ")

From c14e9a308d4d489e4172e025183efabc99b15265 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 16:32:32 +0800
Subject: [PATCH 07/66] test: modify checkpackages scritps

---
 packaging/testpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index c26b2d5e21..ee065da6ec 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -230,7 +230,7 @@ fi
 
 cd ${installPath}
 
-if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [ ${packgeName} =~ "server" ]]) ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .

From bed51f207af28256a52414268caeb2f087cc5c6f Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 18:36:05 +0800
Subject: [PATCH 08/66] test: modify checkpackages scritps

---
 packaging/debRpmAutoInstall.sh | 2 ++
 packaging/testpackage.sh       | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh
index 1f51378c91..3579f813e5 100755
--- a/packaging/debRpmAutoInstall.sh
+++ b/packaging/debRpmAutoInstall.sh
@@ -11,3 +11,5 @@ expect "*one:"
 send  "\r"
 expect "*skip:"
 send  "\r" 
+
+expect eof
\ No newline at end of file
diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index ee065da6ec..d544840082 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -230,7 +230,7 @@ fi
 
 cd ${installPath}
 
-if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .

From aeaba78a8b0a74e0665d63def667a8fc1d17833b Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 21:05:16 +0800
Subject: [PATCH 09/66] test: modify checkpackages scritps of client

---
 packaging/checkPackageRuning.py | 21 ++++++++++++++-------
 packaging/testpackage.sh        |  8 ++++++++
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/packaging/checkPackageRuning.py b/packaging/checkPackageRuning.py
index 39839452b6..2edeeb6dbb 100755
--- a/packaging/checkPackageRuning.py
+++ b/packaging/checkPackageRuning.py
@@ -19,6 +19,13 @@ import subprocess
 # from this import d
 import time
 
+
+if( len(sys.argv)>1 ):
+    serverHost=sys.argv[1]
+else:
+    serverHost="localhost"
+
+
 # install taospy
 
 out = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ")
@@ -32,19 +39,19 @@ else:
 
 
 # start taosd prepare 
-os.system("rm -rf /var/lib/taos/*")
-os.system("systemctl restart taosd ")
+# os.system("rm -rf /var/lib/taos/*")
+# os.system("systemctl restart taosd ")
 
 # wait a moment ,at least 5 seconds
 time.sleep(5)
 
 # prepare data by taosBenchmark 
 
-os.system("taosBenchmark -y -n 100 -t 100")
+os.system("taosBenchmark -y -n 100 -t 100 -h %s "%serverHost )
 
 import taos
 
-conn = taos.connect(host="localhost",
+conn = taos.connect(host="%s"%serverHost,
                                          user="root",
                                          password="taosdata",
                                          database="test",
@@ -80,15 +87,15 @@ os.system("rm -rf /tmp/dumpdata/*")
 # dump data out 
 print("taosdump dump out data")
 
-os.system("taosdump -o /tmp/dumpdata -D test -y ")
+os.system("taosdump -o /tmp/dumpdata -D test -y -h %s  "%serverHost)
 
 # drop database of test
 print("drop database test")
-os.system(" taos -s ' drop database test ;' ")
+os.system(" taos -s ' drop database test ;'  -h %s  "%serverHost)
 
 # dump data in 
 print("taosdump dump data in")
-os.system("taosdump -i /tmp/dumpdata  -y ")
+os.system("taosdump -i /tmp/dumpdata  -y  -h %s  "%serverHost)
 
 result = conn.query("SELECT count(*) from test.meters")
 
diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index d544840082..dce38ce483 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -238,6 +238,14 @@ if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ $
     cd taosTools-2.1.2 && bash install-taostools.sh
 fi
 
+echoColor G  "===== start TDengine ====="
+
+if [[ ${packgeName} =~ "server" ]] ;then
+    echoColor BD " rm -rf /var/lib/taos/* &&  systemctl restart taosd "
+    rm -rf /var/lib/taos/*
+    systemctl restart taosd
+
+
 # if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
 #     echoColor G "===== install taos-tools when package is lite or client ====="
 #     cd ${installPath}

From 76128ee54835602c946d528c0131ed152a62272b Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 21:07:46 +0800
Subject: [PATCH 10/66] test: modify checkpackages scritps of client

---
 packaging/testpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index dce38ce483..03f665d0cb 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -244,7 +244,7 @@ if [[ ${packgeName} =~ "server" ]] ;then
     echoColor BD " rm -rf /var/lib/taos/* &&  systemctl restart taosd "
     rm -rf /var/lib/taos/*
     systemctl restart taosd
-
+fi
 
 # if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
 #     echoColor G "===== install taos-tools when package is lite or client ====="

From 6b107a8522818cceabdd1a809698dea2a7f44791 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 22:40:05 +0800
Subject: [PATCH 11/66] test:update piplines of mptest

---
 packaging/MPtestJenkinsfile | 107 ++++++++++++++++++++----------------
 1 file changed, 61 insertions(+), 46 deletions(-)

diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile
index 77f642180a..5b79345916 100644
--- a/packaging/MPtestJenkinsfile
+++ b/packaging/MPtestJenkinsfile
@@ -5,13 +5,6 @@ def sync_source(branch_name) {
         echo ''' + branch_name + '''
     '''
     sh '''
-        cd ${TDINTERNAL_ROOT_DIR}
-        git reset --hard
-        git fetch || git fetch
-        git checkout ''' + branch_name + ''' -f
-        git branch
-        git pull || git pull
-        git log | head -n 20
         cd ${TDENGINE_ROOT_DIR}
         git reset --hard
         git fetch || git fetch
@@ -64,17 +57,12 @@ pipeline {
             defaultValue:'2.1.2', 
             description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1'
         )
-        string (
-            name:'nasPassword', 
-            defaultValue:'password', 
-            description: 'the pasword of the NAS server which has installPackage-192.168.1.131'
-        )
     }
     environment{
         WORK_DIR = '/var/lib/jenkins/workspace'
         TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal'
         TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community'
-        BRANCH_NAME = '3.0'
+        BRANCH_NAME = 'test/chr/TD-14699'
 
         TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz"
         BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz"
@@ -107,7 +95,7 @@ pipeline {
 
     }
     stages {
-        stage ('RUN') {
+        stage ('Test Server') {
             parallel {
                 stage('ubuntu16') {
                     agent{label " ubuntu16 "}
@@ -116,17 +104,17 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -139,24 +127,21 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} 
-                            python3 checkPackageRuning.py  
-                            '''
-                            sh '''
-                            cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server  
                             python3 checkPackageRuning.py  
+                            dpkg -r tdengine
                             '''
+
                         }
                     }
                 }
@@ -167,17 +152,17 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -190,28 +175,23 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging 
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} 
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server  
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
-                            '''            
-                            sh '''
-                            cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${nasPassword}
-                            python3 checkPackageRuning.py  
-                            '''         
+                            sudo rpm -e tdengine 
+                            '''              
                         }
                     }
-                }
-                                    
+                }                  
                 stage('arm64') {
                     agent{label 'linux_arm64'}
                     steps {
@@ -219,18 +199,53 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
-                            sh '''
-                            cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client  ${nasPassword}
-                            python3 checkPackageRuning.py  
-                            '''                     
                         }
                     }
                 }
             }
         }
-    }
+        stage ('Test Client') {
+            parallel {
+                stage('ubuntu18') {
+                    agent{label " ubuntu18 "}
+                    steps {
+                        timeout(time: 30, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client 
+                            python3 checkPackageRuning.py  192.168.0.21
+                            '''
+                        }
+                    }
+                }
+                stage('centos8') {
+                    agent{label " centos8_3 "}
+                    steps {
+                        timeout(time: 30, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client 
+                            python3 checkPackageRuning.py  192.168.0.24
+                            '''         
+                        }
+                    }
+                }      
+            }
+        }  
+        stage('arm64-client') {
+            agent{label " linux_arm64 "}
+            steps {
+                timeout(time: 30, unit: 'MINUTES'){
+                    sh '''
+                    cd ${TDENGINE_ROOT_DIR}/packaging
+                    bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client  
+                    python3 checkPackageRuning.py  192.168.0.21
+                    '''  
+                }
+            }
+        } 
+    }    
 }
\ No newline at end of file

From 5674429475a05dc3cf945342232d37172ff9b4ae Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 22:54:50 +0800
Subject: [PATCH 12/66] test: modify checkpackages scritps of arm64

---
 packaging/testpackage.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index 03f665d0cb..d1d1efafed 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -236,6 +236,12 @@ if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ $
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
     tar xf taosTools-2.1.2-Linux-x64.tar.gz
     cd taosTools-2.1.2 && bash install-taostools.sh
+elif  ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "server" ]]);then
+    echoColor G "===== install taos-tools when package is lite or client ====="
+    cd ${installPath}
+    wgetFile taosTools-2.1.2-Linux-arm64.tar.gz .
+    tar xf taosTools-2.1.2-Linux-arm64.tar.gz
+    cd taosTools-2.1.2 && bash install-taostools.sh
 fi
 
 echoColor G  "===== start TDengine ====="

From 27e1e332b14a901b0972f0b6d3ddcc315adf795e Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 23:16:40 +0800
Subject: [PATCH 13/66] test: modify checkpackages scritps of arm64

---
 packaging/testpackage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index d1d1efafed..2101ddbf1c 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -236,7 +236,7 @@ if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ $
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
     tar xf taosTools-2.1.2-Linux-x64.tar.gz
     cd taosTools-2.1.2 && bash install-taostools.sh
-elif  ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "server" ]]);then
+elif  ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-arm64.tar.gz .

From 07dd0e30defa40701d958c596a33b3afaaf7b6f8 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 1 Sep 2022 23:36:56 +0800
Subject: [PATCH 14/66] test: modify checkpackages scritps of arm64

---
 packaging/testpackage.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh
index 2101ddbf1c..c018a81158 100755
--- a/packaging/testpackage.sh
+++ b/packaging/testpackage.sh
@@ -230,14 +230,14 @@ fi
 
 cd ${installPath}
 
-if [[ ${packgeName} =~ "Lite" ]]  ||   [[ ${packgeName} =~ "client" ]] ||  ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) ||  ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
     tar xf taosTools-2.1.2-Linux-x64.tar.gz
     cd taosTools-2.1.2 && bash install-taostools.sh
 elif  ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then
-    echoColor G "===== install taos-tools when package is lite or client ====="
+    echoColor G "===== install taos-tools arm when package is arm64-client ====="
     cd ${installPath}
     wgetFile taosTools-2.1.2-Linux-arm64.tar.gz .
     tar xf taosTools-2.1.2-Linux-arm64.tar.gz

From dc4504b4c260ff2ed10548de371d31bda902f894 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Mon, 5 Sep 2022 15:09:23 +0800
Subject: [PATCH 15/66] feat: support table name prefix/suffix config

---
 include/common/tmsg.h                     |  2 ++
 include/libs/qcom/query.h                 |  2 ++
 source/client/src/clientHb.c              |  2 ++
 source/common/src/tmsg.c                  |  4 ++++
 source/dnode/mnode/impl/src/mndDb.c       |  4 ++++
 source/libs/catalog/src/ctgDbg.c          |  8 ++++++--
 source/libs/catalog/src/ctgUtil.c         | 12 ++----------
 source/libs/catalog/test/catalogTests.cpp |  2 ++
 source/libs/qcom/src/querymsg.c           |  2 ++
 9 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index d826f312e5..9f919fa250 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -845,6 +845,8 @@ typedef struct {
   int64_t uid;
   int32_t vgVersion;
   int32_t vgNum;
+  int16_t hashPrefix;
+  int16_t hashSuffix;
   int8_t  hashMethod;
   SArray* pVgroupInfos;  // Array of SVgroupInfo
 } SUseDbRsp;
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index 1fa7dca7dc..44a9e10679 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -116,6 +116,8 @@ typedef struct STableMeta {
 
 typedef struct SDBVgInfo {
   int32_t   vgVersion;
+  int16_t   hashPrefix;
+  int16_t   hashSuffix;
   int8_t    hashMethod;
   int32_t   numOfTable;  // DB's table num, unit is TSDB_TABLE_NUM_UNIT
   SHashObj* vgHash;      // key:vgId, value:SVgroupInfo
diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index 56e3527f96..84a827ed78 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -73,6 +73,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
 
       vgInfo->vgVersion = rsp->vgVersion;
       vgInfo->hashMethod = rsp->hashMethod;
+      vgInfo->hashPrefix = rsp->hashPrefix;
+      vgInfo->hashSuffix = rsp->hashSuffix;
       vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
       if (NULL == vgInfo->vgHash) {
         taosMemoryFree(vgInfo);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index c08817aaf4..ea25094d10 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -2463,6 +2463,8 @@ int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
   if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1;
+  if (tEncodeI16(pEncoder, pRsp->hashPrefix) < 0) return -1;
+  if (tEncodeI16(pEncoder, pRsp->hashSuffix) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1;
 
   for (int32_t i = 0; i < pRsp->vgNum; ++i) {
@@ -2514,6 +2516,8 @@ int32_t tDeserializeSUseDbRspImp(SDecoder *pDecoder, SUseDbRsp *pRsp) {
   if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1;
+  if (tDecodeI16(pDecoder, &pRsp->hashPrefix) < 0) return -1;
+  if (tDecodeI16(pDecoder, &pRsp->hashSuffix) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1;
 
   if (pRsp->vgNum <= 0) {
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 77c9d0bb79..939439ca56 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -1171,6 +1171,8 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs
   pRsp->vgVersion = pDb->vgVersion;
   pRsp->vgNum = taosArrayGetSize(pRsp->pVgroupInfos);
   pRsp->hashMethod = pDb->cfg.hashMethod;
+  pRsp->hashPrefix = pDb->cfg.hashPrefix;
+  pRsp->hashSuffix = pDb->cfg.hashSuffix;
   return 0;
 }
 
@@ -1303,6 +1305,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
     usedbRsp.vgVersion = pDb->vgVersion;
     usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos);
     usedbRsp.hashMethod = pDb->cfg.hashMethod;
+    usedbRsp.hashPrefix = pDb->cfg.hashPrefix;
+    usedbRsp.hashSuffix = pDb->cfg.hashSuffix;
 
     taosArrayPush(batchUseRsp.pArray, &usedbRsp);
     mndReleaseDb(pMnode, pDb);
diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c
index bd3402dc39..d215242307 100644
--- a/source/libs/catalog/src/ctgDbg.c
+++ b/source/libs/catalog/src/ctgDbg.c
@@ -367,18 +367,22 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
     int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0;
     int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
     int32_t hashMethod = -1;
+    int16_t hashPrefix = 0;
+    int16_t hashSuffix = 0;
     int32_t vgNum = 0;
 
     if (dbCache->vgCache.vgInfo) {
       vgVersion = dbCache->vgCache.vgInfo->vgVersion;
       hashMethod = dbCache->vgCache.vgInfo->hashMethod;
+      hashPrefix = dbCache->vgCache.vgInfo->hashPrefix;
+      hashSuffix = dbCache->vgCache.vgInfo->hashSuffix;
       if (dbCache->vgCache.vgInfo->vgHash) {
         vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash);
       }
     }
     
-    ctgDebug("[%d] db [%.*s][0x%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
-      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
+    ctgDebug("[%d] db [%.*s][0x%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d",
+      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, hashPrefix, hashSuffix, vgNum);
 
     pIter = taosHashIterate(dbHash, pIter);
   }
diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c
index e28234ab76..296100ce6d 100644
--- a/source/libs/catalog/src/ctgUtil.c
+++ b/source/libs/catalog/src/ctgUtil.c
@@ -848,15 +848,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName
     CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
   }
 
-  tableNameHashFp fp = NULL;
   SVgroupInfo *vgInfo = NULL;
-
-  CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
-
   char tbFullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pTableName, tbFullName);
 
-  uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
+  uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix);
 
   void *pIter = taosHashIterate(dbInfo->vgHash, NULL);
   while (pIter) {
@@ -919,11 +915,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
     CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
   }
 
-  tableNameHashFp fp = NULL;
   SVgroupInfo *vgInfo = NULL;
-
-  CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
-
   int32_t tbNum = taosArrayGetSize(pNames);
 
   if (1 == vgNum) {
@@ -975,7 +967,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo
     tbNameLen = offset + strlen(pName->tname);
     strcpy(tbFullName + offset, pName->tname);
 
-    uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen);
+    uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix);
 
     SVgroupInfo **p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ);
 
diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp
index 0be85333dc..c01c269e64 100644
--- a/source/libs/catalog/test/catalogTests.cpp
+++ b/source/libs/catalog/test/catalogTests.cpp
@@ -218,6 +218,8 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) {
   ctgTestCurrentVgVersion = dbVgroup->vgVersion;
 
   dbVgroup->hashMethod = 0;
+  dbVgroup->hashPrefix = 0;
+  dbVgroup->hashSuffix = 0;
   dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
 
   vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion);
diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c
index e2d3ac1583..e54937114c 100644
--- a/source/libs/qcom/src/querymsg.c
+++ b/source/libs/qcom/src/querymsg.c
@@ -38,6 +38,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) {
 
   pOut->dbVgroup->vgVersion = usedbRsp->vgVersion;
   pOut->dbVgroup->hashMethod = usedbRsp->hashMethod;
+  pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix;
+  pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix;
 
   qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db);
 

From 31fb2a31b6067b17b1d18845381809fb3f8a1fc0 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Mon, 5 Sep 2022 16:25:13 +0800
Subject: [PATCH 16/66] enh: drop child tables support for rsma

---
 source/dnode/vnode/src/inc/vnodeInt.h   |  4 ++--
 source/dnode/vnode/src/meta/metaTable.c |  6 +++++-
 source/dnode/vnode/src/sma/smaRollup.c  | 14 +++++++-------
 source/dnode/vnode/src/tsdb/tsdbWrite.c |  2 +-
 source/dnode/vnode/src/vnd/vnodeSvr.c   |  9 +++++++--
 5 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 93f9691675..62aba649a3 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -104,7 +104,7 @@ int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq*
 int             metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp);
-int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids);
+int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t *tbUid);
 int             metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids);
 int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
@@ -208,7 +208,7 @@ int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq);
 int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);
 int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq);
 int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
-int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
+int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd);
 void    tdUidStoreDestory(STbUidStore* pStore);
 void*   tdUidStoreFree(STbUidStore* pStore);
 
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 583a2e098f..2c11b9bf0f 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -474,7 +474,7 @@ _err:
   return -1;
 }
 
-int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
+int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) {
   void    *pData = NULL;
   int      nData = 0;
   int      rc = 0;
@@ -496,6 +496,10 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
     taosArrayPush(tbUids, &uid);
   }
 
+  if ((type == TSDB_CHILD_TABLE) && tbUid) {
+    *tbUid = uid;
+  }
+
   tdbFree(pData);
   return 0;
 }
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index fce244bfb8..dc7244ef10 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -34,7 +34,7 @@ typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
 static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
-static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd);
 static int32_t    tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
                                           int8_t idx);
 static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo,
@@ -175,7 +175,7 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
+static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd) {
   SRSmaInfo *pRSmaInfo = NULL;
 
   if (!suid || !tbUids) {
@@ -199,7 +199,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pRSmaInfo->taskInfo[i]) {
-      if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true)) < 0)) {
+      if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0)) {
         tdReleaseRSmaInfo(pSma, pRSmaInfo);
         smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i,
                  terrstr());
@@ -215,12 +215,12 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) {
+int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) {
   if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) {
+  if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids, isAdd) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_FAILED;
   }
 
@@ -229,7 +229,7 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) {
     tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
     SArray   *pTbUids = *(SArray **)pIter;
 
-    if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) {
+    if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) {
       taosHashCancelIterate(pStore->uidHash, pIter);
       return TSDB_CODE_FAILED;
     }
@@ -1118,7 +1118,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
         goto _err;
       }
 
-      if (tdUpdateTbUidList(pVnode->pSma, &uidStore) < 0) {
+      if (tdUpdateTbUidList(pVnode->pSma, &uidStore, true) < 0) {
         smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid,
                  terrstr());
         goto _err;
diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c
index 383652531e..0a9fbf92a4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbWrite.c
+++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c
@@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *
     SSubmitBlkRsp r = {0};
     tGetSubmitMsgNext(&msgIter, &pBlock);
     if (pBlock == NULL) break;
-    if (tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r) < 0) {
+    if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) {
       return -1;
     }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 2724d4cbfd..2d455654a8 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -534,7 +534,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR
   }
 
   tqUpdateTbUidList(pVnode->pTq, tbUids, true);
-  if (tdUpdateTbUidList(pVnode->pSma, pStore) < 0) {
+  if (tdUpdateTbUidList(pVnode->pSma, pStore, true) < 0) {
     goto _exit;
   }
   tdUidStoreFree(pStore);
@@ -692,6 +692,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
   SEncoder         encoder = {0};
   int32_t          ret;
   SArray          *tbUids = NULL;
+  STbUidStore     *pStore = NULL;
 
   pRsp->msgType = TDMT_VND_DROP_TABLE_RSP;
   pRsp->pCont = NULL;
@@ -715,9 +716,10 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
   for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
     SVDropTbReq *pDropTbReq = req.pReqs + iReq;
     SVDropTbRsp  dropTbRsp = {0};
+    tb_uid_t     tbUid = 0;
 
     /* code */
-    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids);
+    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, VND_IS_RSMA(pVnode) ? &tbUid : NULL);
     if (ret < 0) {
       if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) {
         dropTbRsp.code = TSDB_CODE_SUCCESS;
@@ -726,15 +728,18 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
       }
     } else {
       dropTbRsp.code = TSDB_CODE_SUCCESS;
+      if (tbUid > 0) tdFetchTbUidList(pVnode->pSma, &pStore, pDropTbReq->suid, tbUid);
     }
 
     taosArrayPush(rsp.pArray, &dropTbRsp);
   }
 
   tqUpdateTbUidList(pVnode->pTq, tbUids, false);
+  tdUpdateTbUidList(pVnode->pSma, pStore, false);
 
 _exit:
   taosArrayDestroy(tbUids);
+  tdUidStoreFree(pStore);
   tDecoderClear(&decoder);
   tEncodeSize(tEncodeSVDropTbBatchRsp, &rsp, pRsp->contLen, ret);
   pRsp->pCont = rpcMallocCont(pRsp->contLen);

From d8ccd78707d7099ac46dd69f8a656b8118906092 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Mon, 5 Sep 2022 18:02:35 +0800
Subject: [PATCH 17/66] other: code optimization when drop table

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 2d455654a8..51d83d8eed 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -719,7 +719,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq
     tb_uid_t     tbUid = 0;
 
     /* code */
-    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, VND_IS_RSMA(pVnode) ? &tbUid : NULL);
+    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid);
     if (ret < 0) {
       if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) {
         dropTbRsp.code = TSDB_CODE_SUCCESS;

From a6cece10df9f918e02be1919f01fc3c7673cfd10 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 5 Sep 2022 18:57:46 +0800
Subject: [PATCH 18/66] fix(query): fix the invalid the check of last file
 blocks.

---
 source/dnode/vnode/src/inc/tsdb.h           |  2 +-
 source/dnode/vnode/src/tsdb/tsdbCache.c     |  6 ++-
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 56 ++++++++++++++-------
 source/dnode/vnode/src/tsdb/tsdbRead.c      | 32 ++++++------
 4 files changed, 61 insertions(+), 35 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 7546b0943e..35ee8419f6 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -633,7 +633,7 @@ typedef struct SMergeTree {
   struct SLDataIter *pIter;
 } SMergeTree;
 
-int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange);
+int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t suid, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange);
 void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter);
 bool tMergeTreeNext(SMergeTree *pMTree);
 TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 64caff1542..9ac62b4b59 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -420,6 +420,7 @@ typedef enum {
 typedef struct {
   SFSLASTNEXTROWSTATES state;  // [input]
   STsdb               *pTsdb;  // [input]
+  tb_uid_t             suid;
   tb_uid_t             uid;
   int32_t              nFileSet;
   int32_t              iFileSet;
@@ -454,7 +455,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
       code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet);
       if (code) goto _err;
 
-      tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->uid,
+      tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX},
                      &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX});
       bool hasVal = tMergeTreeNext(&state->mergeTree);
@@ -796,7 +797,7 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) {
     if (key->ts > pItemBack->ts) {
       return false;
     } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) {
-      if ((key->version <= pItemFront->version || key->ts == pItemBack->ts && key->version <= pItemBack->version)) {
+      if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) {
         return true;
       } else {
         return false;
@@ -890,6 +891,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
   pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS;
   pIter->fsLastState.pTsdb = pTsdb;
   pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet;
+  pIter->fsLastState.suid = suid;
   pIter->fsLastState.uid = uid;
 
   pIter->fsState.state = SFSNEXTROW_FS;
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index e85b136c2a..9b3f638977 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -42,8 +42,8 @@ static SBlockData* getNextBlock(SLDataIter* pIter) {
   return getCurrentBlock(pIter);
 }
 
-int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t uid,
-                       STimeWindow *pTimeWindow, SVersionRange *pRange) {
+int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t suid,
+                       uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) {
   int32_t code = 0;
   *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
   if (*pIter == NULL) {
@@ -52,11 +52,11 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   }
 
   (*pIter)->uid = uid;
-  (*pIter)->timeWindow = *pTimeWindow;
-  (*pIter)->verRange = *pRange;
-  (*pIter)->pReader = pReader;
   (*pIter)->iSst = iSst;
+  (*pIter)->pReader = pReader;
+  (*pIter)->verRange = *pRange;
   (*pIter)->backward = backward;
+  (*pIter)->timeWindow = *pTimeWindow;
   (*pIter)->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
   if ((*pIter)->aSstBlk == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -85,6 +85,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   if (!backward) {  // asc
     for (int32_t i = 0; i < size; ++i) {
       SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
+      if (p->suid != suid) {
+        continue;
+      }
+
       if (p->minUid <= uid && p->maxUid >= uid) {
         index = i;
         break;
@@ -93,6 +97,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   } else {  // desc
     for (int32_t i = size - 1; i >= 0; --i) {
       SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
+      if (p->suid != suid) {
+        continue;
+      }
+
       if (p->minUid <= uid && p->maxUid >= uid) {
         index = i;
         break;
@@ -134,9 +142,31 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
       break;
     }
 
+    // check uid firstly
     if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) {
-      index = i;
-      break;
+      if ((!pIter->backward) && p->minKey > pIter->timeWindow.ekey) {
+        break;
+      }
+
+      if (pIter->backward && p->maxKey < pIter->timeWindow.skey) {
+        break;
+      }
+
+      // check time range secondly
+      if (p->minKey <= pIter->timeWindow.ekey && p->maxKey >= pIter->timeWindow.skey) {
+        if ((!pIter->backward) && p->minVer > pIter->verRange.maxVer) {
+          break;
+        }
+
+        if (pIter->backward && p->maxVer < pIter->verRange.minVer) {
+          break;
+        }
+
+        if (p->minVer <= pIter->verRange.maxVer && p->maxVer >= pIter->verRange.minVer) {
+          index = i;
+          break;
+        }
+      }
     }
   }
 
@@ -196,14 +226,6 @@ static void findNextValidRow(SLDataIter *pIter) {
       continue;
     }
 
-    //  todo handle delete soon
-#if 0
-    TSDBKEY k = {.ts = ts, .version = ver};
-        if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
-          continue;
-        }
-#endif
-
     hasVal = true;
     break;
   }
@@ -294,7 +316,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
   }
 }
 
-int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t uid,
+int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
                        STimeWindow *pTimeWindow, SVersionRange *pVerRange) {
   pMTree->backward = backward;
   pMTree->pIter = NULL;
@@ -308,7 +330,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
 
   struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0};
   for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) {  // open all last file
-    code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, uid, pTimeWindow, pVerRange);
+    code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 1828f1e40e..ad7f542368 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1934,7 +1934,7 @@ static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockS
   pLastBlockReader->uid = pBlockScanInfo->uid;
   int32_t code =
       tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
-                     pBlockScanInfo->uid, &pLastBlockReader->window, &pLastBlockReader->verRange);
+          pReader->suid, pBlockScanInfo->uid, &pLastBlockReader->window, &pLastBlockReader->verRange);
   if (code != TSDB_CODE_SUCCESS) {
     return false;
   }
@@ -2537,22 +2537,24 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
         bool hasNext = blockIteratorNext(&pReader->status.blockIter);
         if (hasNext) {  // check for the next block in the block accessed order list
           initBlockDumpInfo(pReader, pBlockIter);
-        } else if (hasDataInLastBlock(pReader->status.fileIter.pLastBlockReader)) {
-          // data blocks in current file are exhausted, let's try the next file now
-          tBlockDataReset(&pReader->status.fileBlockData);
-          resetDataBlockIterator(pBlockIter, pReader->order);
-          goto _begin;
         } else {
-          code = initForFirstBlockInFile(pReader, pBlockIter);
-
-          // error happens or all the data files are completely checked
-          if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) {
-            return code;
-          }
-
-          // this file does not have blocks, let's start check the last block file
-          if (pBlockIter->numOfBlocks == 0) {
+          if (pReader->status.pCurrentFileset->nSstF > 0) {
+            // data blocks in current file are exhausted, let's try the next file now
+            tBlockDataReset(&pReader->status.fileBlockData);
+            resetDataBlockIterator(pBlockIter, pReader->order);
             goto _begin;
+          } else {
+            code = initForFirstBlockInFile(pReader, pBlockIter);
+
+            // error happens or all the data files are completely checked
+            if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) {
+              return code;
+            }
+
+            // this file does not have blocks, let's start check the last block file
+            if (pBlockIter->numOfBlocks == 0) {
+              goto _begin;
+            }
           }
         }
       }

From 5701ff674896f5caa3de3f5ed73ec3b20db81494 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 5 Sep 2022 19:03:55 +0800
Subject: [PATCH 19/66] fix(query): fix error in cmakelist file

---
 examples/c/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt
index 2ee63f45c8..4a9007acec 100644
--- a/examples/c/CMakeLists.txt
+++ b/examples/c/CMakeLists.txt
@@ -53,7 +53,7 @@ IF (TD_LINUX)
       )
 
   target_include_directories(demoapi
-      PUBLIC "${TD_SOURCE_DIR}/include/client"other:merge 3.0
+      PUBLIC "${TD_SOURCE_DIR}/include/client"
       PUBLIC "${TD_SOURCE_DIR}/include/os"
       PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
       )

From ef66da9e18aad5a09a10eab3c056f79dcde9c47f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 5 Sep 2022 19:04:53 +0800
Subject: [PATCH 20/66] other:merge 3.0

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 97672b9469..a072f22fa9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -53,6 +53,8 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   (*pIter)->pReader = pReader;
   (*pIter)->iStt = iStt;
   (*pIter)->backward = backward;
+  (*pIter)->verRange = *pRange;
+  (*pIter)->timeWindow = *pTimeWindow;
   (*pIter)->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
   if ((*pIter)->aSttBlk == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;

From e179c10c8895d96fbac9166bacc891b4a201fc43 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 00:26:38 +0800
Subject: [PATCH 21/66] fix(query): set correct last block reader range.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index edd9088974..e0e2272081 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -235,7 +235,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
 
       ASSERT(info.lastKey >= pTsdbReader->window.skey && info.lastKey <= pTsdbReader->window.ekey);
     } else {
-      info.lastKey = pTsdbReader->window.skey;
+      info.lastKey = pTsdbReader->window.ekey;
     }
 
     taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
@@ -1430,6 +1430,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
 
         tRowMergerInit(&merge, &fRow, pReader->pSchema);
         doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+
+        TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+        tRowMerge(&merge, &fRow1);
+
         doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge);
 
         int32_t code = tRowMergerGetRow(&merge, &pTSRow);
@@ -1850,6 +1854,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
 static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
+  int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order)? 1:-1;
+
   while (1) {
     bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
     if (!hasVal) {
@@ -1859,6 +1865,7 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
     TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
     TSDBKEY k = TSDBROW_KEY(&row);
     if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
+      pBlockScanInfo->lastKey = k.ts + step;
       return true;
     }
   }
@@ -1932,9 +1939,17 @@ static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockS
 
   initMemDataIterator(pBlockScanInfo, pReader);
   pLastBlockReader->uid = pBlockScanInfo->uid;
+
+  STimeWindow w = pLastBlockReader->window;
+  if (ASCENDING_TRAVERSE(pLastBlockReader->order)) {
+    w.skey = pBlockScanInfo->lastKey;
+  } else {
+    w.ekey = pBlockScanInfo->lastKey;
+  }
+
   int32_t code =
       tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
-          pReader->suid, pBlockScanInfo->uid, &pLastBlockReader->window, &pLastBlockReader->verRange);
+          pReader->suid, pBlockScanInfo->uid, &w, &pLastBlockReader->verRange);
   if (code != TSDB_CODE_SUCCESS) {
     return false;
   }

From 581cb0ffe1d68fe5406a964724b328549a4b3629 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 01:30:15 +0800
Subject: [PATCH 22/66] other:merge 3.0

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 72 +++-----------------------
 1 file changed, 8 insertions(+), 64 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index e0e2272081..f5d55b5bda 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -229,10 +229,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
     if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
-      if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
-        info.lastKey = pTsdbReader->window.skey;
-      }
-
+      info.lastKey = pTsdbReader->window.skey;
+//      if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
+//        info.lastKey = pTsdbReader->window.skey - step;
+//      }
       ASSERT(info.lastKey >= pTsdbReader->window.skey && info.lastKey <= pTsdbReader->window.ekey);
     } else {
       info.lastKey = pTsdbReader->window.ekey;
@@ -1854,8 +1854,6 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
 static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
-  int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order)? 1:-1;
-
   while (1) {
     bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
     if (!hasVal) {
@@ -1865,65 +1863,9 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
     TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
     TSDBKEY k = TSDBROW_KEY(&row);
     if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
-      pBlockScanInfo->lastKey = k.ts + step;
       return true;
     }
   }
-
-#if 0
-  *(pLastBlockReader->rowIndex) += step;
-
-  SBlockData* pBlockData = &pLastBlockReader->lastBlockData;
-  for (int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) {
-    if (pBlockData->aUid != NULL) {
-      if (asc) {
-        if (pBlockData->aUid[i] < pLastBlockReader->uid) {
-          continue;
-        } else if (pBlockData->aUid[i] > pLastBlockReader->uid) {
-          break;
-        }
-      } else {
-        if (pBlockData->aUid[i] > pLastBlockReader->uid) {
-          continue;
-        } else if (pBlockData->aUid[i] < pLastBlockReader->uid) {
-          break;
-        }
-      }
-    }
-
-    int64_t ts = pBlockData->aTSKEY[i];
-    if (ts < pLastBlockReader->window.skey) {
-      continue;
-    }
-
-    int64_t ver = pBlockData->aVersion[i];
-    if (ver < pLastBlockReader->verRange.minVer) {
-      continue;
-    }
-
-    // no data any more, todo opt handle desc case
-    if (ts > pLastBlockReader->window.ekey) {
-      continue;
-    }
-
-    // todo opt handle desc case
-    if (ver > pLastBlockReader->verRange.maxVer) {
-      continue;
-    }
-
-    TSDBKEY k = {.ts = ts, .version = ver};
-    if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
-      continue;
-    }
-
-    *(pLastBlockReader->rowIndex) = i;
-    return true;
-  }
-
-  // set all data is consumed in last block
-  setAllRowsChecked(pLastBlockReader);
-  return false;
-#endif
 }
 
 static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo,
@@ -1940,11 +1882,12 @@ static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockS
   initMemDataIterator(pBlockScanInfo, pReader);
   pLastBlockReader->uid = pBlockScanInfo->uid;
 
+  int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order)? 1:-1;
   STimeWindow w = pLastBlockReader->window;
   if (ASCENDING_TRAVERSE(pLastBlockReader->order)) {
-    w.skey = pBlockScanInfo->lastKey;
+    w.skey = pBlockScanInfo->lastKey + step;
   } else {
-    w.ekey = pBlockScanInfo->lastKey;
+    w.ekey = pBlockScanInfo->lastKey + step;
   }
 
   int32_t code =
@@ -2915,6 +2858,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
 
 int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
                                SRowMerger* pMerger) {
+  pScanInfo->lastKey = ts;
   while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo)) {
     int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
     if (next1 == ts) {

From a6d9c76681633d524ceda3fc72337c84fd131187 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 01:56:31 +0800
Subject: [PATCH 23/66] fix(query): reset the time range.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index f5d55b5bda..94f587421e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -249,7 +249,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
   return pTableMap;
 }
 
-static void resetDataBlockScanInfo(SHashObj* pTableMap) {
+static void resetDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) {
   STableBlockScanInfo* p = NULL;
 
   while ((p = taosHashIterate(pTableMap, p)) != NULL) {
@@ -260,6 +260,7 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) {
     }
 
     p->delSkyline = taosArrayDestroy(p->delSkyline);
+    p->lastKey = ts;
   }
 }
 
@@ -3550,11 +3551,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
   tsdbDataFReaderClose(&pReader->pFileReader);
 
   int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
-  tsdbDataFReaderClose(&pReader->pFileReader);
 
   initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader);
   resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
-  resetDataBlockScanInfo(pReader->status.pTableMap);
+
+  int64_t ts = ASCENDING_TRAVERSE(pReader->order)?pReader->window.skey:pReader->window.ekey;
+  resetDataBlockScanInfo(pReader->status.pTableMap, ts);
 
   int32_t         code = 0;
   SDataBlockIter* pBlockIter = &pReader->status.blockIter;

From 8f92ffcd3b39060b1a55ddf040dc5215050fba3a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 02:36:04 +0800
Subject: [PATCH 24/66] fix(query): set correct time window.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 94f587421e..8a51fc73a6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -226,16 +226,13 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
     return NULL;
   }
 
+  int32_t step = ASCENDING_TRAVERSE(pTsdbReader->order)? 1:-1;
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
     if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
-      info.lastKey = pTsdbReader->window.skey;
-//      if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
-//        info.lastKey = pTsdbReader->window.skey - step;
-//      }
-      ASSERT(info.lastKey >= pTsdbReader->window.skey && info.lastKey <= pTsdbReader->window.ekey);
+      info.lastKey = pTsdbReader->window.skey - step;
     } else {
-      info.lastKey = pTsdbReader->window.ekey;
+      info.lastKey = pTsdbReader->window.ekey - step;
     }
 
     taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
@@ -3555,7 +3552,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
   initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader);
   resetDataBlockIterator(&pReader->status.blockIter, pReader->order);
 
-  int64_t ts = ASCENDING_TRAVERSE(pReader->order)?pReader->window.skey:pReader->window.ekey;
+  int64_t ts = ASCENDING_TRAVERSE(pReader->order)?pReader->window.skey-1:pReader->window.ekey+1;
   resetDataBlockScanInfo(pReader->status.pTableMap, ts);
 
   int32_t         code = 0;

From 354a62135c84eac2c054faec8c2ef546877823b2 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 6 Sep 2022 09:58:06 +0800
Subject: [PATCH 25/66] other: adjust rsma log level

---
 source/dnode/vnode/src/sma/smaRollup.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index dc7244ef10..fdd173cdf7 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -1701,16 +1701,16 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) {
 
       int64_t curMs = taosGetTimestampMs();
       if ((pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) {
-        smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed",
-                SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay);
+        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed",
+                 SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay);
       } else if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) {
         ++pItem->nSkipped;
-        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ",
+        smaTrace("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ",
                  SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv);
         continue;
       } else {
-        smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ",
-                SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv);
+        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ",
+                 SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv);
       }
 
       pItem->nSkipped = 0;

From 954a1babd33d7c256a912334e258a75946c9857b Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 6 Sep 2022 10:15:10 +0800
Subject: [PATCH 26/66] fix gen-requestId

---
 source/client/src/clientEnv.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index bf92a9ba6a..b739aedca0 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -488,7 +488,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
  */
 uint64_t generateRequestId() {
   static uint64_t hashId = 0;
-  static int32_t  requestSerialId = 0;
+  static uint32_t requestSerialId = 0;
 
   if (hashId == 0) {
     char    uid[64] = {0};
@@ -507,7 +507,8 @@ uint64_t generateRequestId() {
   while (true) {
     int64_t  ts = taosGetTimestampMs();
     uint64_t pid = taosGetPId();
-    int32_t  val = atomic_add_fetch_32(&requestSerialId, 1);
+    uint32_t val = atomic_add_fetch_32(&requestSerialId, 1);
+    if (val >= 0xFFFF) atomic_store_32(&requestSerialId, 0);
 
     id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
     if (id) {

From 863d2aa1f482b33c4726b144465d77e12830ce39 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 6 Sep 2022 11:32:38 +0800
Subject: [PATCH 27/66] fix: invalid read after teardown udfs and close uv pipe
 later

---
 source/libs/function/src/tudf.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 5de9c52cc1..974bcda4d2 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -1183,7 +1183,9 @@ void onUdfcPipeClose(uv_handle_t *handle) {
     QUEUE_REMOVE(&task->procTaskQueue);
     uv_sem_post(&task->taskSem);
   }
-  conn->session->udfUvPipe = NULL;
+  if (conn->session != NULL) {
+    conn->session->udfUvPipe = NULL;
+  }
   taosMemoryFree(conn->readBuf.buf);
   taosMemoryFree(conn);
   taosMemoryFree((uv_pipe_t *) handle);
@@ -1821,7 +1823,10 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
   udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
 
   fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
-
+  if (session->udfUvPipe->data != NULL) {
+    SClientUvConn *conn = session->udfUvPipe->data;
+    conn->session = NULL;
+  }
   taosMemoryFree(session);
   taosMemoryFree(task);
 

From 6593202e320d0e42314c62d3b793d39cbdbaf0c6 Mon Sep 17 00:00:00 2001
From: dapan1121 <72057773+dapan1121@users.noreply.github.com>
Date: Tue, 6 Sep 2022 11:41:05 +0800
Subject: [PATCH 28/66] Update 20-keywords.md

add key usage description
---
 docs/zh/12-taos-sql/20-keywords.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md
index 92c814067e..09fea45b7a 100644
--- a/docs/zh/12-taos-sql/20-keywords.md
+++ b/docs/zh/12-taos-sql/20-keywords.md
@@ -6,7 +6,8 @@ description: TDengine 保留关键字的详细列表
 
 ## 保留关键字
 
-目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下：
+目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写如果需要用作库名、表名、STable 名、数据列名及标签列名等，需要使用符合``将关键字括起来使用，例如`ADD`。
+关键字列表如下：
 
 ### A
 
@@ -239,6 +240,7 @@ description: TDengine 保留关键字的详细列表
 - TOPICS
 - TRIGGER
 - TSERIES
+- TTL
 
 ### U
 

From 63ba9420f37bde858f1c036e7962cdfd36d87d7d Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 6 Sep 2022 13:47:16 +0800
Subject: [PATCH 29/66] fix: change whether uv pipe exists

---
 source/libs/function/src/tudf.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 974bcda4d2..5d894783fb 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -1823,7 +1823,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
   udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
 
   fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
-  if (session->udfUvPipe->data != NULL) {
+  if (session->udfUvPipe != NULL && session->udfUvPipe->data != NULL) {
     SClientUvConn *conn = session->udfUvPipe->data;
     conn->session = NULL;
   }

From 6360f0616b57f567ad0a9db685ecd612877fb733 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 6 Sep 2022 14:48:45 +0800
Subject: [PATCH 30/66] fix: fix create_time column missed issue

---
 source/dnode/mnode/impl/src/mndDb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 939439ca56..c598b188b7 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -1592,6 +1592,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
       SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i);
       if (i == 0) {
         colDataAppend(pColInfo, rows, buf, false);
+      } else if (i == 1) {
+        colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false);
       } else if (i == 3) {
         colDataAppend(pColInfo, rows, (const char *)&numOfTables, false);
       } else if (i == 14) {

From 68a1c12daec428e4101baeaec455dc6c181af6a4 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 6 Sep 2022 14:57:56 +0800
Subject: [PATCH 31/66] add compile opt

---
 source/libs/index/src/indexComm.c     | 12 ++---
 source/libs/index/src/indexFilter.c   | 67 +++++++++++++--------------
 source/libs/transport/src/thttp.c     | 10 ++--
 source/libs/transport/src/transCli.c  | 50 +++++++++-----------
 source/libs/transport/src/transComm.c |  2 +-
 source/libs/transport/src/transSvr.c  | 22 ++++-----
 6 files changed, 78 insertions(+), 85 deletions(-)

diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c
index 691eb6771c..ecf9136073 100644
--- a/source/libs/index/src/indexComm.c
+++ b/source/libs/index/src/indexComm.c
@@ -81,28 +81,28 @@ __compar_fn_t idxGetCompar(int8_t type) {
   }
   return getComparFunc(type, 0);
 }
-static TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
+static FORCE_INLINE TExeCond tCompareLessThan(void* a, void* b, int8_t type) {
   __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_LESS_THAN, a, b, type);
 }
-static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
+static FORCE_INLINE TExeCond tCompareLessEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_LESS_EQUAL, a, b, type);
 }
-static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
+static FORCE_INLINE TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) {
   __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_GREATER_THAN, a, b, type);
 }
-static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
+static FORCE_INLINE TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
 
-static TExeCond tCompareContains(void* a, void* b, int8_t type) {
+static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) {
   __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_TERM, a, b, type);
 }
-static TExeCond tCompareEqual(void* a, void* b, int8_t type) {
+static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) {
   __compar_fn_t func = idxGetCompar(type);
   return tCompare(func, QUERY_TERM, a, b, type);
 }
diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c
index e7f221de3d..b65acc4672 100644
--- a/source/libs/index/src/indexFilter.c
+++ b/source/libs/index/src/indexFilter.c
@@ -88,7 +88,7 @@ typedef struct SIFCtx {
   SIndexMetaArg arg;
 } SIFCtx;
 
-static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
+static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
   if (src == OP_TYPE_GREATER_THAN) {
     *dst = QUERY_GREATER_THAN;
   } else if (src == OP_TYPE_GREATER_EQUAL) {
@@ -110,10 +110,9 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
 }
 
 typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output);
-
 static sif_func_t sifNullFunc = NULL;
 
-static void sifFreeParam(SIFParam *param) {
+static FORCE_INLINE void sifFreeParam(SIFParam *param) {
   if (param == NULL) return;
 
   taosArrayDestroy(param->result);
@@ -123,7 +122,7 @@ static void sifFreeParam(SIFParam *param) {
   param->pFilter = NULL;
 }
 
-static int32_t sifGetOperParamNum(EOperatorType ty) {
+static FORCE_INLINE int32_t sifGetOperParamNum(EOperatorType ty) {
   if (OP_TYPE_IS_NULL == ty || OP_TYPE_IS_NOT_NULL == ty || OP_TYPE_IS_TRUE == ty || OP_TYPE_IS_NOT_TRUE == ty ||
       OP_TYPE_IS_FALSE == ty || OP_TYPE_IS_NOT_FALSE == ty || OP_TYPE_IS_UNKNOWN == ty ||
       OP_TYPE_IS_NOT_UNKNOWN == ty || OP_TYPE_MINUS == ty) {
@@ -131,14 +130,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) {
   }
   return 2;
 }
-static int32_t sifValidOp(EOperatorType ty) {
+static FORCE_INLINE int32_t sifValidOp(EOperatorType ty) {
   if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) ||
       (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) {
     return -1;
   }
   return 0;
 }
-static int32_t sifValidColumn(SColumnNode *cn) {
+static FORCE_INLINE int32_t sifValidColumn(SColumnNode *cn) {
   // add more check
   if (cn == NULL) {
     return TSDB_CODE_QRY_INVALID_INPUT;
@@ -149,7 +148,7 @@ static int32_t sifValidColumn(SColumnNode *cn) {
   return TSDB_CODE_SUCCESS;
 }
 
-static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) {
+static FORCE_INLINE SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) {
   // enh rule later
   if (type == LOGIC_COND_TYPE_AND) {
     if (ls == SFLT_NOT_INDEX || rs == SFLT_NOT_INDEX) {
@@ -167,7 +166,7 @@ static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SI
   return SFLT_NOT_INDEX;
 }
 
-static int32_t sifGetValueFromNode(SNode *node, char **value) {
+static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) {
   // covert data From snode;
   SValueNode *vn = (SValueNode *)node;
 
@@ -205,7 +204,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
+static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   SOperatorNode *nd = (SOperatorNode *)node;
   assert(nodeType(node) == QUERY_NODE_OPERATOR);
   SColumnNode *l = (SColumnNode *)nd->pLeft;
@@ -355,30 +354,30 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
 
-typedef int (*Filter)(void *a, void *b, int16_t dtype);
+typedef int (*FilterFunc)(void *a, void *b, int16_t dtype);
 
-int sifGreaterThan(void *a, void *b, int16_t dtype) {
+static FORCE_INLINE int sifGreaterThan(void *a, void *b, int16_t dtype) {
   __compar_fn_t func = getComparFunc(dtype, 0);
   return tDoCompare(func, QUERY_GREATER_THAN, a, b);
 }
-int sifGreaterEqual(void *a, void *b, int16_t dtype) {
+static FORCE_INLINE int sifGreaterEqual(void *a, void *b, int16_t dtype) {
   __compar_fn_t func = getComparFunc(dtype, 0);
   return tDoCompare(func, QUERY_GREATER_EQUAL, a, b);
 }
-int sifLessEqual(void *a, void *b, int16_t dtype) {
+static FORCE_INLINE int sifLessEqual(void *a, void *b, int16_t dtype) {
   __compar_fn_t func = getComparFunc(dtype, 0);
   return tDoCompare(func, QUERY_LESS_EQUAL, a, b);
 }
-int sifLessThan(void *a, void *b, int16_t dtype) {
+static FORCE_INLINE int sifLessThan(void *a, void *b, int16_t dtype) {
   __compar_fn_t func = getComparFunc(dtype, 0);
   return (int)tDoCompare(func, QUERY_LESS_THAN, a, b);
 }
-int sifEqual(void *a, void *b, int16_t dtype) {
+static FORCE_INLINE int sifEqual(void *a, void *b, int16_t dtype) {
   __compar_fn_t func = getComparFunc(dtype, 0);
   //__compar_fn_t func = idxGetCompar(dtype);
   return (int)tDoCompare(func, QUERY_TERM, a, b);
 }
-static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
+static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
   if (type == QUERY_LESS_EQUAL || type == QUERY_LESS_THAN) {
     *reverse = true;
   } else {
@@ -470,8 +469,8 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
     indexMultiTermQueryAdd(mtm, tm, qtype);
     ret = indexJsonSearch(arg->ivtIdx, mtm, output->result);
   } else {
-    bool   reverse;
-    Filter filterFunc = sifGetFilterFunc(qtype, &reverse);
+    bool       reverse;
+    FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse);
 
     SMetaFltParam param = {.suid = arg->suid,
                            .cid = left->colId,
@@ -498,72 +497,72 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
   return ret;
 }
 
-static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_LOWER_THAN;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_LOWER_EQUAL;
   return sifDoIndex(left, right, id, output);
 }
 
-static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_GREATER_THAN;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_GREATER_EQUAL;
   return sifDoIndex(left, right, id, output);
 }
 
-static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_EQUAL;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_NOT_EQUAL;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_IN;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_NOT_IN;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_LIKE;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_NOT_LIKE;
   return sifDoIndex(left, right, id, output);
 }
 
-static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_MATCH;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_NMATCH;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) {
   int id = OP_TYPE_JSON_CONTAINS;
   return sifDoIndex(left, right, id, output);
 }
-static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) {
+static FORCE_INLINE int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) {
   // return 0
   return 0;
 }
 
-static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+static FORCE_INLINE int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   // add more except
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
 
-static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) {
+static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) {
   // impl later
   *status = SFLT_ACCURATE_INDEX;
   switch (funcId) {
diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c
index 275e7b42cc..1559c85e23 100644
--- a/source/libs/transport/src/thttp.c
+++ b/source/libs/transport/src/thttp.c
@@ -126,22 +126,22 @@ _OVER:
   return code;
 }
 
-static void destroyHttpClient(SHttpClient* cli) {
+static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) {
   taosMemoryFree(cli->wbuf);
   taosMemoryFree(cli->rbuf);
   taosMemoryFree(cli->addr);
   taosMemoryFree(cli);
 }
-static void clientCloseCb(uv_handle_t* handle) {
+static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) {
   SHttpClient* cli = handle->data;
   destroyHttpClient(cli);
 }
-static void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
+static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
   SHttpClient* cli = handle->data;
   buf->base = cli->rbuf;
   buf->len = HTTP_RECV_BUF_SIZE;
 }
-static void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
+static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
   SHttpClient* cli = handle->data;
   if (nread < 0) {
     uError("http-report recv error:%s", uv_err_name(nread));
@@ -173,7 +173,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) {
   uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb);
 }
 
-static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
+static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
   uint32_t ip = taosGetIpv4FromFqdn(server);
   if (ip == 0xffffffff) {
     terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 4b5441f738..e648f14437 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -69,11 +69,9 @@ typedef struct SCliThrd {
   SAsyncPool*   asyncPool;
   uv_prepare_t* prepare;
   void*         pool;  // conn pool
-
+  // timer handles
   SArray* timerList;
-
   // msg queue
-
   queue         msg;
   TdThreadMutex msgMtx;
   SDelayQueue*  delayQueue;
@@ -108,7 +106,7 @@ static void cliReadTimeoutCb(uv_timer_t* handle);
 // register timer in each thread to clear expire conn
 // static void cliTimeoutCb(uv_timer_t* handle);
 // alloc buffer for recv
-static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
+static FORCE_INLINE void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
 // callback after recv nbytes from socket
 static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
 // callback after send data to socket
@@ -132,10 +130,10 @@ static void      cliSend(SCliConn* pConn);
 static void      cliDestroyConnMsgs(SCliConn* conn, bool destroy);
 
 // cli util func
-static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx);
-static void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
+static FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx);
+static FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
 
-static int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp);
+static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp);
 
 // process data read from server, add decompress etc later
 static void cliHandleResp(SCliConn* conn);
@@ -150,12 +148,10 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd);
 static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL,
                                                                    cliHandleUpdate};
 
-static void cliSendQuit(SCliThrd* thrd);
-static void destroyUserdata(STransMsg* userdata);
+static FORCE_INLINE void destroyUserdata(STransMsg* userdata);
+static FORCE_INLINE void destroyCmsg(void* cmsg);
+static FORCE_INLINE int  cliRBChoseIdx(STrans* pTransInst);
 
-static int cliRBChoseIdx(STrans* pTransInst);
-
-static void destroyCmsg(void* cmsg);
 static void transDestroyConnCtx(STransConnCtx* ctx);
 // thread obj
 static SCliThrd* createThrdObj();
@@ -885,26 +881,23 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
   }
   return conn;
 }
-void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
+FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
   if (pCvtAddr->cvt == false) {
     return;
   }
-  for (int i = 0; i < pEpSet->numOfEps && pEpSet->numOfEps == 1; i++) {
-    if (strncmp(pEpSet->eps[i].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) {
-      memset(pEpSet->eps[i].fqdn, 0, TSDB_FQDN_LEN);
-      memcpy(pEpSet->eps[i].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN);
-    }
+  if (pEpSet->numOfEps == 1 && strncmp(pEpSet->eps[0].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) {
+    memset(pEpSet->eps[0].fqdn, 0, TSDB_FQDN_LEN);
+    memcpy(pEpSet->eps[0].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN);
   }
 }
 
-bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
+FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
   if (code != 0) return false;
   if (pCtx->retryCnt == 0) return false;
   if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false;
   return true;
 }
-
-int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
+FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
   if (pMsg == NULL) return -1;
 
   memset(pResp, 0, sizeof(STransMsg));
@@ -1128,14 +1121,15 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
   return cli;
 }
 
-static void destroyUserdata(STransMsg* userdata) {
+FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
   if (userdata->pCont == NULL) {
     return;
   }
   transFreeMsg(userdata->pCont);
   userdata->pCont = NULL;
 }
-static void destroyCmsg(void* arg) {
+
+FORCE_INLINE void destroyCmsg(void* arg) {
   SCliMsg* pMsg = arg;
   if (pMsg == NULL) {
     return;
@@ -1220,7 +1214,7 @@ void cliWalkCb(uv_handle_t* handle, void* arg) {
   }
 }
 
-int cliRBChoseIdx(STrans* pTransInst) {
+FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst) {
   int8_t index = pTransInst->index;
   if (pTransInst->numOfThreads == 0) {
     return -1;
@@ -1230,7 +1224,7 @@ int cliRBChoseIdx(STrans* pTransInst) {
   }
   return index % pTransInst->numOfThreads;
 }
-static void doDelayTask(void* param) {
+static FORCE_INLINE void doDelayTask(void* param) {
   STaskArg* arg = param;
   SCliMsg*  pMsg = arg->param1;
   SCliThrd* pThrd = arg->param2;
@@ -1264,13 +1258,13 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) {
   transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
 }
 
-void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
+FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
   if (*val != exp) {
     *val = newVal;
   }
 }
 
-bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) {
+FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) {
   if ((pResp == NULL || pResp->info.hasEpSet == 0)) {
     return false;
   }
@@ -1402,7 +1396,7 @@ void transUnrefCliHandle(void* handle) {
     cliDestroyConn((SCliConn*)handle, true);
   }
 }
-SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) {
+static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) {
   SCliThrd*  pThrd = NULL;
   SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle);
   if (exh == NULL) {
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index dea96fa3ac..5f3171ee0e 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -424,7 +424,7 @@ void transQueueDestroy(STransQueue* queue) {
   taosArrayDestroy(queue->q);
 }
 
-static int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
+static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
   SDelayTask* arg1 = container_of(a, SDelayTask, node);
   SDelayTask* arg2 = container_of(b, SDelayTask, node);
   if (arg1->execTime > arg2->execTime) {
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 7007079f87..c00cafb708 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -125,17 +125,17 @@ static void uvWorkAfterTask(uv_work_t* req, int status);
 static void uvWalkCb(uv_handle_t* handle, void* arg);
 static void uvFreeCb(uv_handle_t* handle);
 
-static void uvStartSendRespImpl(SSvrMsg* smsg);
+static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg);
+
 static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
 static void uvStartSendResp(SSvrMsg* msg);
 
 static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
 
-static void destroySmsg(SSvrMsg* smsg);
-// check whether already read complete packet
-static SSvrConn* createConn(void* hThrd);
-static void      destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
-static void      destroyConnRegArg(SSvrConn* conn);
+static FORCE_INLINE void      destroySmsg(SSvrMsg* smsg);
+static FORCE_INLINE SSvrConn* createConn(void* hThrd);
+static FORCE_INLINE void      destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
+static FORCE_INLINE void      destroyConnRegArg(SSvrConn* conn);
 
 static int reallocConnRef(SSvrConn* conn);
 
@@ -413,7 +413,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
   wb->len = len;
 }
 
-static void uvStartSendRespImpl(SSvrMsg* smsg) {
+static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
   SSvrConn* pConn = smsg->pConn;
   if (pConn->broken) {
     return;
@@ -447,7 +447,7 @@ static void uvStartSendResp(SSvrMsg* smsg) {
   return;
 }
 
-static void destroySmsg(SSvrMsg* smsg) {
+static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
   if (smsg == NULL) {
     return;
   }
@@ -812,7 +812,7 @@ void* transWorkerThread(void* arg) {
   return NULL;
 }
 
-static SSvrConn* createConn(void* hThrd) {
+static FORCE_INLINE SSvrConn* createConn(void* hThrd) {
   SWorkThrd* pThrd = hThrd;
 
   SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn));
@@ -842,7 +842,7 @@ static SSvrConn* createConn(void* hThrd) {
   return pConn;
 }
 
-static void destroyConn(SSvrConn* conn, bool clear) {
+static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear) {
   if (conn == NULL) {
     return;
   }
@@ -854,7 +854,7 @@ static void destroyConn(SSvrConn* conn, bool clear) {
     }
   }
 }
-static void destroyConnRegArg(SSvrConn* conn) {
+static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn) {
   if (conn->regArg.init == 1) {
     transFreeMsg(conn->regArg.msg.pCont);
     conn->regArg.init = 0;

From a5933fc682d5d560fa0b5381ee839022c6969657 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 15:25:54 +0800
Subject: [PATCH 32/66] fix(query): optimize last block read performance.

---
 source/dnode/vnode/src/inc/tsdb.h           |  16 +-
 source/dnode/vnode/src/tsdb/tsdbCache.c     |   2 +-
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 200 ++++++++++++++------
 source/dnode/vnode/src/tsdb/tsdbRead.c      |  49 ++---
 4 files changed, 182 insertions(+), 85 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index d45a6f19f0..8c7f9be062 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -643,20 +643,34 @@ typedef struct {
   TSDBROW row;
 } SRowInfo;
 
+typedef struct SSttBlockLoadInfo {
+    int32_t    sttFileIndex;
+    SBlockData blockData[2];
+    SArray    *aSttBlk;
+    int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
+    int32_t    currentLoadBlockIndex;
+} SSttBlockLoadInfo;
+
 typedef struct SMergeTree {
   int8_t      backward;
   SRBTree     rbt;
   SArray     *pIterList;
   SLDataIter *pIter;
+  bool        destroyLoadInfo;
+  SSttBlockLoadInfo* pLoadInfo;
 } SMergeTree;
 
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
-                       STimeWindow *pTimeWindow, SVersionRange *pVerRange);
+                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pLoadInfo);
 void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool    tMergeTreeNext(SMergeTree *pMTree);
 TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);
 
+SSttBlockLoadInfo* tCreateLastBlockLoadInfo();
+void  resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo);
+void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo);
+
 // ========== inline functions ==========
 static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
   TSDBKEY *pKey1 = (TSDBKEY *)p1;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 9ac62b4b59..8af764c4bc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -457,7 +457,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
 
       tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX},
-                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX});
+                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL);
       bool hasVal = tMergeTreeNext(&state->mergeTree);
       if (!hasVal) {
         state->state = SFSLASTNEXTROW_FILESET;
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index a072f22fa9..beb0004e24 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -22,26 +22,105 @@ struct SLDataIter {
   SDataFReader *pReader;
   int32_t       iStt;
   int8_t        backward;
-  SArray       *aSttBlk;
   int32_t       iSttBlk;
-  SBlockData    bData[2];
-  int32_t       loadIndex;
   int32_t       iRow;
   SRowInfo      rInfo;
   uint64_t      uid;
   STimeWindow   timeWindow;
   SVersionRange verRange;
+
+  SSttBlockLoadInfo* pBlockLoadInfo;
 };
 
-static SBlockData *getCurrentBlock(SLDataIter *pIter) { return &pIter->bData[pIter->loadIndex]; }
+SSttBlockLoadInfo* tCreateLastBlockLoadInfo() {
+  SSttBlockLoadInfo* pLoadInfo = taosMemoryCalloc(TSDB_DEFAULT_STT_FILE, sizeof(SSttBlockLoadInfo));
+  if (pLoadInfo == NULL) {
+    terrno =  TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
 
-static SBlockData *getNextBlock(SLDataIter *pIter) {
-  pIter->loadIndex ^= 1;
-  return getCurrentBlock(pIter);
+  for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) {
+    pLoadInfo[i].blockIndex[0] = -1;
+    pLoadInfo[i].blockIndex[1] = -1;
+    pLoadInfo[i].currentLoadBlockIndex = 1;
+
+    int32_t code = tBlockDataCreate(&pLoadInfo[i].blockData[0]);
+    if (code) {
+      terrno = code;
+    }
+
+    code = tBlockDataCreate(&pLoadInfo[i].blockData[1]);
+    if (code) {
+      terrno = code;
+    }
+
+    pLoadInfo[i].aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
+  }
+
+  return pLoadInfo;
+}
+
+void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) {
+  for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) {
+    pLoadInfo[i].currentLoadBlockIndex = 1;
+    pLoadInfo[i].blockIndex[0] = -1;
+    pLoadInfo[i].blockIndex[1] = -1;
+
+    taosArrayClear(pLoadInfo[i].aSttBlk);
+  }
+}
+
+void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) {
+  for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) {
+    pLoadInfo[i].currentLoadBlockIndex = 1;
+    pLoadInfo[i].blockIndex[0] = -1;
+    pLoadInfo[i].blockIndex[1] = -1;
+
+    tBlockDataDestroy(&pLoadInfo[i].blockData[0], true);
+    tBlockDataDestroy(&pLoadInfo[i].blockData[1], true);
+
+    taosArrayDestroy(pLoadInfo[i].aSttBlk);
+  }
+
+  taosMemoryFree(pLoadInfo);
+  return NULL;
+}
+
+static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
+  int32_t code = 0;
+
+  SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo;
+  if (pInfo->blockIndex[0]  == pIter->iStt) {
+    return &pInfo->blockData[0];
+  }
+
+  if (pInfo->blockIndex[1] == pIter->iStt) {
+    return &pInfo->blockData[1];
+  }
+
+  pInfo->currentLoadBlockIndex ^= 1;
+  if (pIter->pSttBlk != NULL) {  // current block not loaded yet
+    code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _exit;
+    }
+
+    pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iStt;
+    pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1;
+  }
+
+  return &pInfo->blockData[pInfo->currentLoadBlockIndex];
+
+  _exit:
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+  }
+
+  return NULL;
 }
 
 int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
-    uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) {
+                       uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) {
   int32_t code = 0;
   *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
   if (*pIter == NULL) {
@@ -55,34 +134,35 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   (*pIter)->backward = backward;
   (*pIter)->verRange = *pRange;
   (*pIter)->timeWindow = *pTimeWindow;
-  (*pIter)->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
-  if ((*pIter)->aSttBlk == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
+
+  (*pIter)->pBlockLoadInfo = pBlockLoadInfo;
+
+  if (pBlockLoadInfo->aSttBlk == NULL) {
+    // loaded into the common shared objects
+    pBlockLoadInfo->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+    if (pBlockLoadInfo->aSttBlk == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _exit;
+    }
+
+    code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
+    if (code) {
+      goto _exit;
+    }
   }
 
-  code = tBlockDataCreate(&(*pIter)->bData[0]);
+  code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
   if (code) {
     goto _exit;
   }
 
-  code = tBlockDataCreate(&(*pIter)->bData[1]);
-  if (code) {
-    goto _exit;
-  }
-
-  code = tsdbReadSttBlk(pReader, iStt, (*pIter)->aSttBlk);
-  if (code) {
-    goto _exit;
-  }
-
-  size_t size = taosArrayGetSize((*pIter)->aSttBlk);
+  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
   int32_t index = -1;
   if (!backward) {  // asc
     for (int32_t i = 0; i < size; ++i) {
-      SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i);
+      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
       if (p->suid != suid) {
         continue;
       }
@@ -94,7 +174,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     }
   } else {  // desc
     for (int32_t i = size - 1; i >= 0; --i) {
-      SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i);
+      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
       if (p->suid != suid) {
         continue;
       }
@@ -108,7 +188,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
   (*pIter)->iSttBlk = index;
   if (index != -1) {
-    (*pIter)->pSttBlk = taosArrayGet((*pIter)->aSttBlk, (*pIter)->iSttBlk);
+    (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
   }
 
 _exit:
@@ -116,9 +196,6 @@ _exit:
 }
 
 void tLDataIterClose(SLDataIter *pIter) {
-  tBlockDataDestroy(&pIter->bData[0], 1);
-  tBlockDataDestroy(&pIter->bData[1], 1);
-  taosArrayDestroy(pIter->aSttBlk);
   taosMemoryFree(pIter);
 }
 
@@ -127,9 +204,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
   pIter->iSttBlk += step;
 
   int32_t index = -1;
-  size_t  size = taosArrayGetSize(pIter->aSttBlk);
+  size_t  size = taosArrayGetSize(pIter->pBlockLoadInfo->aSttBlk);
   for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) {
-    SSttBlk *p = taosArrayGet(pIter->aSttBlk, i);
+    SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i);
     if ((!pIter->backward) && p->minUid > pIter->uid) {
       break;
     }
@@ -169,7 +246,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
   if (index == -1) {
     pIter->pSttBlk = NULL;
   } else {
-    pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+    pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
   }
 }
 
@@ -178,7 +255,8 @@ static void findNextValidRow(SLDataIter *pIter) {
 
   bool        hasVal = false;
   int32_t     i = pIter->iRow;
-  SBlockData *pBlockData = getCurrentBlock(pIter);
+
+  SBlockData *pBlockData = loadBlockIfMissing(pIter);
 
   for (; i < pBlockData->nRow && i >= 0; i += step) {
     if (pBlockData->aUid != NULL) {
@@ -238,18 +316,8 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
     return false;
   }
 
-  int32_t     iBlockL = pIter->iSttBlk;
-  SBlockData *pBlockData = getCurrentBlock(pIter);
-
-  if (pBlockData->nRow == 0 && pIter->pSttBlk != NULL) {  // current block not loaded yet
-    pBlockData = getNextBlock(pIter);
-    code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData);
-    if (code != TSDB_CODE_SUCCESS) {
-      goto _exit;
-    }
-
-    pIter->iRow = (pIter->backward) ? pBlockData->nRow : -1;
-  }
+  int32_t iBlockL = pIter->iSttBlk;
+  SBlockData *pBlockData = loadBlockIfMissing(pIter);
 
   pIter->iRow += step;
 
@@ -266,12 +334,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
     }
 
     if (iBlockL != pIter->iSttBlk) {
-      pBlockData = getNextBlock(pIter);
-      code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData);
-      if (code) {
-        goto _exit;
-      }
-      pIter->iRow = pIter->backward ? (pBlockData->nRow - 1) : 0;
+      pBlockData = loadBlockIfMissing(pIter);
     }
   }
 
@@ -313,7 +376,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
 }
 
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
-                       STimeWindow *pTimeWindow, SVersionRange *pVerRange) {
+                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo) {
   pMTree->backward = backward;
   pMTree->pIter = NULL;
   pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
@@ -322,21 +385,33 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
   }
 
   tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
-  int32_t code = TSDB_CODE_OUT_OF_MEMORY;
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  SSttBlockLoadInfo* pLoadInfo = NULL;
+  if (pBlockLoadInfo == NULL) {
+    if (pMTree->pLoadInfo == NULL) {
+      pMTree->destroyLoadInfo = true;
+      pMTree->pLoadInfo = tCreateLastBlockLoadInfo();
+    }
+
+    pLoadInfo = pMTree->pLoadInfo;
+  } else {
+    pLoadInfo = pBlockLoadInfo;
+  }
 
-  struct SLDataIter *pIterList[TSDB_DEFAULT_STT_FILE] = {0};
   for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) {  // open all last file
-    code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange);
+    struct SLDataIter* pIter = NULL;
+    code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pLoadInfo[i]);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
 
-    bool hasVal = tLDataIterNextRow(pIterList[i]);
+    bool hasVal = tLDataIterNextRow(pIter);
     if (hasVal) {
-      taosArrayPush(pMTree->pIterList, &pIterList[i]);
-      tMergeTreeAddIter(pMTree, pIterList[i]);
+      taosArrayPush(pMTree->pIterList, &pIter);
+      tMergeTreeAddIter(pMTree, pIter);
     } else {
-      tLDataIterClose(pIterList[i]);
+      tLDataIterClose(pIter);
     }
   }
 
@@ -393,4 +468,9 @@ void tMergeTreeClose(SMergeTree *pMTree) {
 
   pMTree->pIterList = taosArrayDestroy(pMTree->pIterList);
   pMTree->pIter = NULL;
+
+  if (pMTree->destroyLoadInfo) {
+    pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo);
+    pMTree->destroyLoadInfo = false;
+  }
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 8a51fc73a6..e404dd3d3a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -17,8 +17,6 @@
 #include "tsdb.h"
 
 #define ASCENDING_TRAVERSE(o)  (o == TSDB_ORDER_ASC)
-#define ALL_ROWS_CHECKED_INDEX (INT16_MIN)
-#define INITIAL_ROW_INDEX_VAL  (-1)
 
 typedef enum {
   EXTERNAL_ROWS_PREV = 0x1,
@@ -88,6 +86,7 @@ typedef struct SLastBlockReader {
   int32_t       order;
   uint64_t      uid;
   SMergeTree    mergeTree;
+  SSttBlockLoadInfo* pInfo;
 } SLastBlockReader;
 
 typedef struct SFilesetIter {
@@ -226,13 +225,12 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
     return NULL;
   }
 
-  int32_t step = ASCENDING_TRAVERSE(pTsdbReader->order)? 1:-1;
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
     if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
-      info.lastKey = pTsdbReader->window.skey - step;
+      info.lastKey = pTsdbReader->window.skey - 1;
     } else {
-      info.lastKey = pTsdbReader->window.ekey - step;
+      info.lastKey = pTsdbReader->window.ekey + 1;
     }
 
     taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
@@ -319,8 +317,7 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap
 }
 
 // init file iterator
-static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet,
-                                   STsdbReader* pReader /*int32_t order, const char* idstr*/) {
+static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader) {
   size_t numOfFileset = taosArrayGetSize(aDFileSet);
 
   pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset;
@@ -345,6 +342,12 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet,
   pLReader->uid = 0;
   tMergeTreeClose(&pLReader->mergeTree);
 
+  pLReader->pInfo = tCreateLastBlockLoadInfo();
+  if (pLReader->pInfo == NULL) {
+    tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
+    return terrno;
+  }
+
   tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr);
   return TSDB_CODE_SUCCESS;
 }
@@ -360,6 +363,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) {
 
   pIter->pLastBlockReader->uid = 0;
   tMergeTreeClose(&pIter->pLastBlockReader->mergeTree);
+  resetLastBlockLoadInfo(pIter->pLastBlockReader->pInfo);
 
   // check file the time range of coverage
   STimeWindow win = {0};
@@ -1377,7 +1381,6 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
                                             STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,
                                             bool mergeBlockData) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
-  // SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData;
   int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
 
   STSRow*    pTSRow = NULL;
@@ -1866,36 +1869,35 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc
   }
 }
 
-static bool initLastBlockReader(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo,
-                                STsdbReader* pReader) {
+static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
   // the last block reader has been initialized for this table.
-  if (pLastBlockReader->uid == pBlockScanInfo->uid) {
+  if (pLBlockReader->uid == pScanInfo->uid) {
     return true;
   }
 
-  if (pLastBlockReader->uid != 0) {
-    tMergeTreeClose(&pLastBlockReader->mergeTree);
+  if (pLBlockReader->uid != 0) {
+    tMergeTreeClose(&pLBlockReader->mergeTree);
   }
 
-  initMemDataIterator(pBlockScanInfo, pReader);
-  pLastBlockReader->uid = pBlockScanInfo->uid;
+  initMemDataIterator(pScanInfo, pReader);
+  pLBlockReader->uid = pScanInfo->uid;
 
-  int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order)? 1:-1;
-  STimeWindow w = pLastBlockReader->window;
-  if (ASCENDING_TRAVERSE(pLastBlockReader->order)) {
-    w.skey = pBlockScanInfo->lastKey + step;
+  int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order)? 1:-1;
+  STimeWindow w = pLBlockReader->window;
+  if (ASCENDING_TRAVERSE(pLBlockReader->order)) {
+    w.skey = pScanInfo->lastKey + step;
   } else {
-    w.ekey = pBlockScanInfo->lastKey + step;
+    w.ekey = pScanInfo->lastKey + step;
   }
 
   int32_t code =
-      tMergeTreeOpen(&pLastBlockReader->mergeTree, (pLastBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
-          pReader->suid, pBlockScanInfo->uid, &w, &pLastBlockReader->verRange);
+      tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
+          pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo);
   if (code != TSDB_CODE_SUCCESS) {
     return false;
   }
 
-  return nextRowFromLastBlocks(pLastBlockReader, pBlockScanInfo);
+  return nextRowFromLastBlocks(pLBlockReader, pScanInfo);
 }
 
 static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
@@ -3305,6 +3307,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
   SFilesetIter* pFilesetIter = &pReader->status.fileIter;
   if (pFilesetIter->pLastBlockReader != NULL) {
     tMergeTreeClose(&pFilesetIter->pLastBlockReader->mergeTree);
+    pFilesetIter->pLastBlockReader->pInfo = destroyLastBlockLoadInfo(pFilesetIter->pLastBlockReader->pInfo);
     taosMemoryFree(pFilesetIter->pLastBlockReader);
   }
 

From 84f79e8df2e56fd8fb95f2a5abb2a1a9f3c07e8d Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 6 Sep 2022 15:48:08 +0800
Subject: [PATCH 33/66] fix: save selectivity value for first/last even column
 consists of all null values

---
 source/libs/function/src/builtinsimpl.c | 14 ++++++++++++--
 source/libs/function/src/tudf.c         |  2 ++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index dcf58759b0..f396f6e609 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2795,6 +2795,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
   // All null data column, return directly.
   if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
     ASSERT(pInputCol->hasNull == true);
+    // save selectivity value for column consisted of all null values
+    firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);
     return 0;
   }
 
@@ -2871,7 +2873,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
     }
   }
 #endif
-
+  if (numOfElems == 0) {
+    // save selectivity value for column consisted of all null values
+    firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);
+  }
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
 }
@@ -2892,6 +2897,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
   // All null data column, return directly.
   if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) {
     ASSERT(pInputCol->hasNull == true);
+    // save selectivity value for column consisted of all null values
+    firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);
     return 0;
   }
 
@@ -2952,7 +2959,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
     }
   }
 #endif
-
+  if (numOfElems == 0) {
+    // save selectivity value for column consisted of all null values
+    firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo);
+  }
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
 }
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 5d894783fb..3f472b53a0 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -1805,6 +1805,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
 
   if (session->udfUvPipe == NULL) {
     fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName);
+    taosMemoryFree(session);
     return TSDB_CODE_UDF_PIPE_NO_PIPE;
   }
 
@@ -1823,6 +1824,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
   udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
 
   fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
+  //TODO: synchronization refactor between libuv event loop and request thread
   if (session->udfUvPipe != NULL && session->udfUvPipe->data != NULL) {
     SClientUvConn *conn = session->udfUvPipe->data;
     conn->session = NULL;

From bdb7f20d3dc4007ecdd3a1533896bd9538f910d2 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 6 Sep 2022 16:29:59 +0800
Subject: [PATCH 34/66] fix: force pageSize as i64 to fix u32 overflow

---
 source/libs/tdb/src/db/tdbPager.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index f90c392788..bec07b935e 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -502,7 +502,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
   i64 offset;
   int ret;
 
-  offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
+  offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
   if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
     ASSERT(0);
     return -1;

From 6f252d8d7a3300b8cf1d7350e143ffd81ae9cc8a Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 16:36:56 +0800
Subject: [PATCH 35/66] fix(query): opt perf on last block query.

---
 source/dnode/vnode/src/inc/tsdb.h           |  1 -
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 22 +++++----------------
 2 files changed, 5 insertions(+), 18 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 8c7f9be062..8bca22fea1 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -644,7 +644,6 @@ typedef struct {
 } SRowInfo;
 
 typedef struct SSttBlockLoadInfo {
-    int32_t    sttFileIndex;
     SBlockData blockData[2];
     SArray    *aSttBlk;
     int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index beb0004e24..4347550c2d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -90,22 +90,23 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
   int32_t code = 0;
 
   SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo;
-  if (pInfo->blockIndex[0]  == pIter->iStt) {
+  if (pInfo->blockIndex[0]  == pIter->iSttBlk) {
     return &pInfo->blockData[0];
   }
 
-  if (pInfo->blockIndex[1] == pIter->iStt) {
+  if (pInfo->blockIndex[1] == pIter->iSttBlk) {
     return &pInfo->blockData[1];
   }
 
   pInfo->currentLoadBlockIndex ^= 1;
   if (pIter->pSttBlk != NULL) {  // current block not loaded yet
     code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]);
+    tsdbDebug("read last block, index:%d, last file index:%d", pIter->iSttBlk, pIter->iStt);
     if (code != TSDB_CODE_SUCCESS) {
       goto _exit;
     }
 
-    pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iStt;
+    pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk;
     pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1;
   }
 
@@ -136,26 +137,13 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   (*pIter)->timeWindow = *pTimeWindow;
 
   (*pIter)->pBlockLoadInfo = pBlockLoadInfo;
-
-  if (pBlockLoadInfo->aSttBlk == NULL) {
-    // loaded into the common shared objects
-    pBlockLoadInfo->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
-    if (pBlockLoadInfo->aSttBlk == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _exit;
-    }
-
+  if (taosArrayGetSize(pBlockLoadInfo->aSttBlk) == 0) {
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
       goto _exit;
     }
   }
 
-  code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
-  if (code) {
-    goto _exit;
-  }
-
   size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block

From a929f837d4ce32a1d2077351679b0b07dc5f173f Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 6 Sep 2022 17:31:19 +0800
Subject: [PATCH 36/66] fix: rsma qtaskfile restore

---
 source/dnode/vnode/src/sma/smaRollup.c | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index fdd173cdf7..14bf479533 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -1342,29 +1342,31 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
     return TSDB_CODE_FAILED;
   }
 
-  if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) {
+  if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) {
     return TSDB_CODE_FAILED;
   }
 
   int32_t infoLen = 0;
-  taosDecodeFixedI32(pIter->qBuf, &infoLen);
+  taosDecodeFixedI32(pIter->pBuf, &infoLen);
   if (infoLen > nBytes) {
     if (infoLen <= RSMA_QTASKINFO_BUFSIZE) {
       terrno = TSDB_CODE_RSMA_FILE_CORRUPTED;
       smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
       return TSDB_CODE_FAILED;
     }
-    pIter->nAlloc = infoLen;
-    void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
-    if (!pBuf) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return TSDB_CODE_FAILED;
+    if (pIter->nAlloc < infoLen) {
+      pIter->nAlloc = infoLen;
+      void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
+      if (!pBuf) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return TSDB_CODE_FAILED;
+      }
+      pIter->pBuf = pBuf;
     }
-    pIter->pBuf = pBuf;
-    pIter->qBuf = pIter->pBuf;
+
     nBytes = infoLen;
 
-    if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
+    if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
       return TSDB_CODE_FAILED;
     }
 
@@ -1373,6 +1375,7 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
     }
   }
 
+  pIter->qBuf = pIter->pBuf;
   pIter->offset += nBytes;
   pIter->nBytes = nBytes;
   pIter->nBufPos = 0;

From aacf318ba09f1bbd2a682becafdead201cf8fd5e Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 6 Sep 2022 18:18:34 +0800
Subject: [PATCH 37/66] fix: qtaskinfo fs open for rsma

---
 source/dnode/vnode/src/sma/smaFS.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c
index 4387db553e..9c3a0a0b9e 100644
--- a/source/dnode/vnode/src/sma/smaFS.c
+++ b/source/dnode/vnode/src/sma/smaFS.c
@@ -49,7 +49,7 @@ int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) {
   for (int32_t i = 0; i < taosArrayGetSize(output); ++i) {
     int32_t vid = 0;
     int64_t version = -1;
-    sscanf((const char *)taosArrayGetP(output, i), "v%dqinfo.v%" PRIi64, &vid, &version);
+    sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version);
     SQTaskFile qTaskFile = {.version = version, .nRef = 1};
     if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) {
       goto _end;

From d1755fad2421027c56165b5bb3d86200259bb437 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 6 Sep 2022 18:47:43 +0800
Subject: [PATCH 38/66] fix trans cli mem leakk

---
 source/libs/transport/src/transCli.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index e648f14437..f744327a59 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -430,6 +430,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
 
     if (pCtx == NULL || pCtx->pSem == NULL) {
       if (transMsg.info.ahandle == NULL) {
+        destroyCmsg(pMsg);
         once = true;
         continue;
       }
@@ -973,6 +974,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
       return;
     }
   }
+  STraceId* trace = &pMsg->msg.info.traceId;
+  tGTrace("%s conn %p ready", pTransInst->label, conn);
 }
 static void cliAsyncCb(uv_async_t* handle) {
   SAsyncItem* item = handle->data;
@@ -1294,7 +1297,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
   STrans*   pTransInst = pThrd->pTransInst;
 
   if (pMsg == NULL || pMsg->ctx == NULL) {
-    tTrace("%s conn %p handle resp", pTransInst->label, pConn);
+    tDebug("%s conn %p handle resp", pTransInst->label, pConn);
     pTransInst->cfp(pTransInst->parent, pResp, NULL);
     return 0;
   }

From 6c40af0059a6453eb4d89c7e9fd12baa92b31128 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 6 Sep 2022 19:16:56 +0800
Subject: [PATCH 39/66] fix no resp error

---
 source/libs/transport/src/transCli.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index f744327a59..ab1fcc31dc 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -435,7 +435,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
         continue;
       }
     }
-    if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
+    if (!REQUEST_NO_RESP(&pMsg->msg) && cliAppCb(pConn, &transMsg, pMsg) != 0) {
       return;
     }
     destroyCmsg(pMsg);

From 4a563b6d50c310077c1c52c5933b6065cb32fcad Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 6 Sep 2022 19:17:11 +0800
Subject: [PATCH 40/66] fix: fix now in alter tag value

---
 source/libs/executor/src/executil.c    |  1 -
 source/libs/parser/src/parTranslater.c | 23 ++++++++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 3965b7e5b2..69951e5e1c 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -475,7 +475,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
     qError("failed to create result, reason:%s", tstrerror(code));
-    terrno = code;
     goto end;
   }
 
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 240aa0d6c0..bebc2e3e1d 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -6603,11 +6603,32 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   }
   pReq->colId = pSchema->colId;
 
+  SValueNode *pVal = NULL;
+  
   SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
-  if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
+  pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal);
+  if (pCxt->errCode) {
     return pCxt->errCode;
   }
 
+/*  
+  if (QUERY_NODE_FUNCTION == pStmt->pVal->node.type) {
+    pCxt->errCode = getFuncInfo(pCxt, (SFunctionNode*)pStmt->pVal);
+    if (pCxt->errCode) {
+      return pCxt->errCode;
+    }
+    
+    if (DEAL_RES_ERROR == translateFunctionImpl(pCxt, (SFunctionNode**)&pStmt->pVal)) {
+      return pCxt->errCode;
+    }
+  } else if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
+    return pCxt->errCode;
+  }
+*/
+
+  nodesDestroyNode((SNode*)pStmt->pVal);
+
+  pStmt->pVal = pVal;
   pReq->tagType = targetDt.type;
   if (targetDt.type == TSDB_DATA_TYPE_JSON) {
     if (pStmt->pVal->literal &&

From cf46b22267f282c8cce335e533432510f0eeca9b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 6 Sep 2022 19:27:49 +0800
Subject: [PATCH 41/66] fix(query): fix error in tsdbread.

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c |  3 ++-
 source/dnode/vnode/src/tsdb/tsdbRead.c      | 16 ++++++++++------
 tests/script/tsim/parser/commit.sim         |  1 +
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 4347550c2d..950c9348af 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -177,6 +177,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   (*pIter)->iSttBlk = index;
   if (index != -1) {
     (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
+    (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1;
   }
 
 _exit:
@@ -306,7 +307,6 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
 
   int32_t iBlockL = pIter->iSttBlk;
   SBlockData *pBlockData = loadBlockIfMissing(pIter);
-
   pIter->iRow += step;
 
   while (1) {
@@ -323,6 +323,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
 
     if (iBlockL != pIter->iSttBlk) {
       pBlockData = loadBlockIfMissing(pIter);
+      pIter->iRow += step;
     }
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index e404dd3d3a..facfdb9a62 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -228,9 +228,11 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
     if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
-      info.lastKey = pTsdbReader->window.skey - 1;
+      int64_t skey = pTsdbReader->window.skey;
+      info.lastKey = (skey > INT64_MIN)? (skey - 1):skey;
     } else {
-      info.lastKey = pTsdbReader->window.ekey + 1;
+      int64_t ekey = pTsdbReader->window.ekey;
+      info.lastKey = (ekey < INT64_MAX)? (ekey + 1):ekey;
     }
 
     taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info));
@@ -342,10 +344,12 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
   pLReader->uid = 0;
   tMergeTreeClose(&pLReader->mergeTree);
 
-  pLReader->pInfo = tCreateLastBlockLoadInfo();
   if (pLReader->pInfo == NULL) {
-    tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
-    return terrno;
+    pLReader->pInfo = tCreateLastBlockLoadInfo();
+    if (pLReader->pInfo == NULL) {
+      tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
+      return terrno;
+    }
   }
 
   tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr);
@@ -1892,7 +1896,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
 
   int32_t code =
       tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader,
-          pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo);
+                     pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo);
   if (code != TSDB_CODE_SUCCESS) {
     return false;
   }
diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim
index ae19a4803b..a9bf8b26eb 100644
--- a/tests/script/tsim/parser/commit.sim
+++ b/tests/script/tsim/parser/commit.sim
@@ -97,6 +97,7 @@ while $loop <= $loops
   endw
   sql select count(*) from $stb
   if $data00 != $totalNum then
+    print expect $totalNum , actual: $data00
     return -1
   endi
   $loop = $loop + 1

From b277748662879e2ab0b257370207a7ca21f1190e Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 6 Sep 2022 20:09:12 +0800
Subject: [PATCH 42/66] fix mem leak

---
 source/libs/transport/src/transCli.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index ab1fcc31dc..dc2d937c49 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -430,12 +430,12 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
 
     if (pCtx == NULL || pCtx->pSem == NULL) {
       if (transMsg.info.ahandle == NULL) {
-        destroyCmsg(pMsg);
+        if (REQUEST_NO_RESP(&pMsg->msg)) destroyCmsg(pMsg);
         once = true;
         continue;
       }
     }
-    if (!REQUEST_NO_RESP(&pMsg->msg) && cliAppCb(pConn, &transMsg, pMsg) != 0) {
+    if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
       return;
     }
     destroyCmsg(pMsg);

From b15fa09acd957cf62a2a1057cfb9784bd9b4c4fe Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Tue, 6 Sep 2022 20:11:50 +0800
Subject: [PATCH 43/66] enh: qinf process optimization for duplicated commit
 version

---
 source/dnode/vnode/src/inc/sma.h       |  1 +
 source/dnode/vnode/src/sma/smaCommit.c | 21 ++++++++++++++++-----
 source/dnode/vnode/src/sma/smaFS.c     | 12 ++++++++++++
 source/dnode/vnode/src/sma/smaRollup.c | 17 ++++++++++++-----
 4 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h
index c9d1c3d015..165f5da4b6 100644
--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@@ -211,6 +211,7 @@ int32_t tdRSmaFSOpen(SSma *pSma, int64_t version);
 void    tdRSmaFSClose(SRSmaFS *fs);
 int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version);
 void    tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version);
+int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat);
 int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile);
 int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer);
 int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c
index fd300ef34a..f08cefdb04 100644
--- a/source/dnode/vnode/src/sma/smaCommit.c
+++ b/source/dnode/vnode/src/sma/smaCommit.c
@@ -182,6 +182,7 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) {
   SVnode  *pVnode = pSma->pVnode;
   SRSmaFS *pFS = RSMA_FS(pStat);
   int64_t  committed = pStat->commitAppliedVer;
+  int64_t  fsMaxVer = -1;
   char     qTaskInfoFullName[TSDB_FILENAME_LEN];
 
   taosWLockLatch(RSMA_FS_LOCK(pStat));
@@ -204,10 +205,20 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) {
     ++i;
   }
 
-  SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0};
-  if (tdRSmaFSUpsertQTaskFile(pFS, &qFile) < 0) {
-    taosWUnLockLatch(RSMA_FS_LOCK(pStat));
-    return TSDB_CODE_FAILED;
+  if (taosArrayGetSize(pFS->aQTaskInf) > 0) {
+    fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version;
+  }
+
+  if (fsMaxVer < committed) {
+    SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0};
+    if (taosArrayPush(pFS->aQTaskInf, &qFile) < 0) {
+      taosWUnLockLatch(RSMA_FS_LOCK(pStat));
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+  } else {
+    smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode),
+            committed, fsMaxVer);
   }
 
   taosWUnLockLatch(RSMA_FS_LOCK(pStat));
@@ -365,7 +376,7 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
     return TSDB_CODE_SUCCESS;
   }
 
-  SRSmaStat     *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
 
   // step 1: merge qTaskInfo and iQTaskInfo
   // lock
diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c
index 9c3a0a0b9e..8e8611f0e8 100644
--- a/source/dnode/vnode/src/sma/smaFS.c
+++ b/source/dnode/vnode/src/sma/smaFS.c
@@ -96,6 +96,18 @@ int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) {
   return oldVal;
 }
 
+int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) {
+  SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf;
+  int64_t version = -1;
+
+  taosRLockLatch(RSMA_FS_LOCK(pStat));
+  if (taosArrayGetSize(aQTaskInf) > 0) {
+    version = ((SQTaskFile *)taosArrayGetLast(aQTaskInf))->version;
+  }
+  taosRUnLockLatch(RSMA_FS_LOCK(pStat));
+  return version;
+}
+
 void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) {
   SVnode     *pVnode = pSma->pVnode;
   SArray     *aQTaskInf = RSMA_FS(pStat)->aQTaskInf;
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 14bf479533..920d04bf6c 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -1453,17 +1453,24 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
     return TSDB_CODE_SUCCESS;
   }
 
+  int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat);
+  if (pRSmaStat->commitAppliedVer <= fsMaxVer) {
+    smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid,
+            pRSmaStat->commitAppliedVer, fsMaxVer);
+    return TSDB_CODE_SUCCESS;
+  }
+
   STFile tFile = {0};
 #if 0
   if (pRSmaStat->commitAppliedVer > 0) {
     char qTaskInfoFName[TSDB_FILENAME_LEN];
     tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName);
     if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
-      smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
+      smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr());
       goto _err;
     }
     if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) {
-      smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
+      smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
       goto _err;
     }
     smaDebug("vgId:%d, rsma, serialize qTaskInfo, file %s created", vid, TD_TFILE_FULL_NAME(&tFile));
@@ -1513,11 +1520,11 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
         char qTaskInfoFName[TSDB_FILENAME_LEN];
         tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName);
         if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
-          smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
+          smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr());
           goto _err;
         }
         if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) {
-          smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
+          smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
           goto _err;
         }
         smaDebug("vgId:%d, rsma, table %" PRIi64 " serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid,
@@ -1561,7 +1568,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
   }
   return TSDB_CODE_SUCCESS;
 _err:
-  smaError("vgId:%d, rsma persit failed since %s", vid, terrstr());
+  smaError("vgId:%d, rsma persist failed since %s", vid, terrstr());
   if (isFileCreated) {
     tdRemoveTFile(&tFile);
     tdDestroyTFile(&tFile);

From 7c3c988b4c39789620a5699cc08a15f1ecbdaca3 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Tue, 6 Sep 2022 22:53:36 +0800
Subject: [PATCH 44/66] build: release ver-3.0.0.2

---
 cmake/cmake.version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/cmake.version b/cmake/cmake.version
index db29644b38..0873e59e92 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.0.1")
+  SET(TD_VER_NUMBER "3.0.0.2")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

From 577aee289b2ff4bc2809b140399681455e948718 Mon Sep 17 00:00:00 2001
From: Linhe Huo <linhehuo@gmail.com>
Date: Wed, 7 Sep 2022 02:30:30 +0800
Subject: [PATCH 45/66] chore: update taosws (#16699)

Fix [TD-18952](https://jira.taosdata.com:18080/browse/TD-18952)
---
 cmake/taosws_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in
index 54e898ecc0..15d3764876 100644
--- a/cmake/taosws_CMakeLists.txt.in
+++ b/cmake/taosws_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taosws-rs
 ExternalProject_Add(taosws-rs
         GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
-        GIT_TAG 6fc47d7
+        GIT_TAG 0609b50
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 851ffe49aea50de8d2dd02eb7bff3788ddb2cf60 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 7 Sep 2022 10:25:21 +0800
Subject: [PATCH 46/66] enh: support set tag value with expr

---
 source/libs/parser/src/parTranslater.c | 23 ++++++-----------------
 source/util/test/pageBufferTest.cpp    |  8 ++++----
 2 files changed, 10 insertions(+), 21 deletions(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index bebc2e3e1d..ad9a467dee 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -6603,32 +6603,21 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   }
   pReq->colId = pSchema->colId;
 
-  SValueNode *pVal = NULL;
-  
   SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
-  pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal);
-  if (pCxt->errCode) {
-    return pCxt->errCode;
-  }
 
-/*  
-  if (QUERY_NODE_FUNCTION == pStmt->pVal->node.type) {
-    pCxt->errCode = getFuncInfo(pCxt, (SFunctionNode*)pStmt->pVal);
+  if (QUERY_NODE_VALUE != pStmt->pVal->node.type) {
+    SValueNode *pVal = NULL;
+    pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal);
     if (pCxt->errCode) {
       return pCxt->errCode;
     }
-    
-    if (DEAL_RES_ERROR == translateFunctionImpl(pCxt, (SFunctionNode**)&pStmt->pVal)) {
-      return pCxt->errCode;
-    }
+
+    nodesDestroyNode((SNode*)pStmt->pVal);
+    pStmt->pVal = pVal;
   } else if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
     return pCxt->errCode;
   }
-*/
 
-  nodesDestroyNode((SNode*)pStmt->pVal);
-
-  pStmt->pVal = pVal;
   pReq->tagType = targetDt.type;
   if (targetDt.type == TSDB_DATA_TYPE_JSON) {
     if (pStmt->pVal->literal &&
diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp
index 1a057c5875..534c177587 100644
--- a/source/util/test/pageBufferTest.cpp
+++ b/source/util/test/pageBufferTest.cpp
@@ -23,9 +23,9 @@ void simpleTest() {
   
   ASSERT_EQ(getTotalBufSize(pBuf), 1024);
   
-  SIDList list = getDataBufPagesIdList(pBuf, groupId);
+  SIDList list = getDataBufPagesIdList(pBuf);
   ASSERT_EQ(taosArrayGetSize(list), 1);
-  ASSERT_EQ(getNumOfBufGroupId(pBuf), 1);
+  //ASSERT_EQ(getNumOfBufGroupId(pBuf), 1);
 
   releaseBufPage(pBuf, pBufPage);
 
@@ -98,7 +98,7 @@ void writeDownTest() {
   SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pBuf, writePageId));
   ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
 
-  SArray* pa = getDataBufPagesIdList(pBuf, groupId);
+  SArray* pa = getDataBufPagesIdList(pBuf);
   ASSERT_EQ(taosArrayGetSize(pa), 5);
 
   destroyDiskbasedBuf(pBuf);
@@ -152,7 +152,7 @@ void recyclePageTest() {
 
   SFilePage* pBufPagex1 = static_cast<SFilePage*>(getBufPage(pBuf, 1));
 
-  SArray* pa = getDataBufPagesIdList(pBuf, groupId);
+  SArray* pa = getDataBufPagesIdList(pBuf);
   ASSERT_EQ(taosArrayGetSize(pa), 6);
 
   destroyDiskbasedBuf(pBuf);

From c151dd3b925c1d21780f1437e61f71f7fb4471c1 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 7 Sep 2022 10:35:44 +0800
Subject: [PATCH 47/66] fix(sync): mem leak

---
 source/libs/sync/src/syncMain.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 51098374b0..6f29b54f80 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -2766,8 +2766,6 @@ const char* syncStr(ESyncState state) {
 }
 
 int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) {
-  SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg);
-
   if (ths->state != TAOS_SYNC_STATE_FOLLOWER) {
     syncNodeEventLog(ths, "I am not follower, can not do leader transfer");
     return 0;
@@ -2799,6 +2797,8 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p
     }
   */
 
+  SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg);
+
   do {
     char logBuf[128];
     snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index);

From 8d49a840eb9ee73ef5f4a16588a2a1f560ca7335 Mon Sep 17 00:00:00 2001
From: dapan1121 <72057773+dapan1121@users.noreply.github.com>
Date: Wed, 7 Sep 2022 11:08:18 +0800
Subject: [PATCH 48/66] fix: remove column comments

---
 docs/zh/12-taos-sql/03-table.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md
index 2337f2a272..f6790e3c69 100644
--- a/docs/zh/12-taos-sql/03-table.md
+++ b/docs/zh/12-taos-sql/03-table.md
@@ -23,10 +23,7 @@ create_subtable_clause: {
 }
 
 create_definition:
-    col_name column_definition
-
-column_definition:
-    type_name [comment 'string_value']
+    col_name column_type
 
 table_options:
     table_option ...

From 6a862b71990bdcceece800061138bcf58f4e37d7 Mon Sep 17 00:00:00 2001
From: afwerar <1296468573@qq.com>
Date: Wed, 7 Sep 2022 11:34:55 +0800
Subject: [PATCH 49/66] build: release ver-3.0.1.0

---
 cmake/cmake.version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/cmake.version b/cmake/cmake.version
index 0873e59e92..7c895b12ff 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.0.2")
+  SET(TD_VER_NUMBER "3.0.1.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

From e7149cce3a96120c0f14beb0fc9219ad1d0696c0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 7 Sep 2022 13:18:34 +0800
Subject: [PATCH 50/66] fix(query): opt perf in query last file blocks.

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 134 +++++++++++++++-----
 source/dnode/vnode/src/tsdb/tsdbRead.c      | 108 +++++++++++-----
 2 files changed, 174 insertions(+), 68 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index 950c9348af..a721cfdbfc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -120,6 +120,92 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
   return NULL;
 }
 
+// find the earliest block that contains the required records
+static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t backward) {
+  int32_t i = index;
+  int32_t step = backward? 1:-1;
+  while (uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid && i >= 0) {
+    i += step;
+  }
+  return i - step;
+}
+
+static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64_t uid, int32_t backward) {
+  int32_t midPos = -1;
+  if (num <= 0) {
+    return -1;
+  }
+
+  int32_t firstPos = 0;
+  int32_t lastPos = num - 1;
+
+  // find the first position which is bigger than the key
+  if ((uid > pBlockList[lastPos].maxUid) || (uid < pBlockList[firstPos].minUid)) {
+    return -1;
+  }
+
+  while (1) {
+    if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) {
+      return findEarliestIndex(firstPos, uid, pBlockList, backward);
+    }
+
+    if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) {
+      return -1;
+    }
+
+    int32_t numOfRows = lastPos - firstPos + 1;
+    midPos = (numOfRows >> 1u) + firstPos;
+
+    if (uid < pBlockList[midPos].minUid) {
+      lastPos = midPos - 1;
+    } else if (uid > pBlockList[midPos].maxUid) {
+      firstPos = midPos + 1;
+    } else {
+      return findEarliestIndex(midPos, uid, pBlockList, backward);
+    }
+  }
+}
+
+static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t backward) {
+  int32_t i = index;
+  int32_t step = backward? 1:-1;
+  while (uid == uidList[i] && i >= 0) {
+    i += step;
+  }
+  return i - step;
+}
+
+static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint64_t uid, int32_t backward) {
+  int32_t firstPos = 0;
+  int32_t lastPos = num - 1;
+
+  // find the first position which is bigger than the key
+  if ((uid > uidList[lastPos]) || (uid < uidList[firstPos])) {
+    return -1;
+  }
+
+  while (1) {
+    if (uid == uidList[firstPos]) {
+      return findEarliestRow(firstPos, uid, uidList, backward);
+    }
+
+    if (uid > uidList[lastPos] || uid < uidList[firstPos]) {
+      return -1;
+    }
+
+    int32_t numOfRows = lastPos - firstPos + 1;
+    int32_t midPos = (numOfRows >> 1u) + firstPos;
+
+    if (uid < uidList[midPos]) {
+      lastPos = midPos - 1;
+    } else if (uid > uidList[midPos]) {
+      firstPos = midPos + 1;
+    } else {
+      return findEarliestRow(midPos, uid, uidList, backward);
+    }
+  }
+}
+
 int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
                        uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) {
   int32_t code = 0;
@@ -147,33 +233,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
-  int32_t index = -1;
-  if (!backward) {  // asc
-    for (int32_t i = 0; i < size; ++i) {
-      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
-      if (p->suid != suid) {
-        continue;
-      }
-
-      if (p->minUid <= uid && p->maxUid >= uid) {
-        index = i;
-        break;
-      }
-    }
-  } else {  // desc
-    for (int32_t i = size - 1; i >= 0; --i) {
-      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
-      if (p->suid != suid) {
-        continue;
-      }
-
-      if (p->minUid <= uid && p->maxUid >= uid) {
-        index = i;
-        break;
-      }
-    }
-  }
-
+  int32_t index = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
   (*pIter)->iSttBlk = index;
   if (index != -1) {
     (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
@@ -193,7 +253,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
   pIter->iSttBlk += step;
 
   int32_t index = -1;
-  size_t  size = taosArrayGetSize(pIter->pBlockLoadInfo->aSttBlk);
+  size_t  size = pIter->pBlockLoadInfo->aSttBlk->size;//taosArrayGetSize(pIter->pBlockLoadInfo->aSttBlk);
   for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) {
     SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i);
     if ((!pIter->backward) && p->minUid > pIter->uid) {
@@ -232,9 +292,8 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
     }
   }
 
-  if (index == -1) {
-    pIter->pSttBlk = NULL;
-  } else {
+  pIter->pSttBlk = NULL;
+  if (index != -1) {
     pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
   }
 }
@@ -247,18 +306,23 @@ static void findNextValidRow(SLDataIter *pIter) {
 
   SBlockData *pBlockData = loadBlockIfMissing(pIter);
 
+  // mostly we only need to find the start position for a given table
+  if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && pBlockData->aUid != NULL) {
+    i = binarySearchForStartRowIndex((uint64_t*)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward);
+  }
+
   for (; i < pBlockData->nRow && i >= 0; i += step) {
     if (pBlockData->aUid != NULL) {
       if (!pIter->backward) {
-        if (pBlockData->aUid[i] < pIter->uid) {
+        /*if (pBlockData->aUid[i] < pIter->uid) {
           continue;
-        } else if (pBlockData->aUid[i] > pIter->uid) {
+        } else */if (pBlockData->aUid[i] > pIter->uid) {
           break;
         }
       } else {
-        if (pBlockData->aUid[i] > pIter->uid) {
+        /*if (pBlockData->aUid[i] > pIter->uid) {
           continue;
-        } else if (pBlockData->aUid[i] < pIter->uid) {
+        } else */if (pBlockData->aUid[i] < pIter->uid) {
           break;
         }
       }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index facfdb9a62..d205ca13e2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1238,6 +1238,38 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
   return false;
 }
 
+static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
+  while (1) {
+    bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
+    if (!hasVal) {
+      return false;
+    }
+
+    TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+    TSDBKEY k = TSDBROW_KEY(&row);
+    if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
+      return true;
+    }
+  }
+}
+
+static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader,
+                                           STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) {
+  bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo);
+  if (hasVal) {
+    int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
+    if (next1 != ts) {
+      doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
+      return true;
+    }
+  } else {
+    doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow);
+    return true;
+  }
+
+  return false;
+}
+
 static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) {
   // always set the newest schema version in pReader->pSchema
   if (pReader->pSchema == NULL) {
@@ -1385,29 +1417,55 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
                                             STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,
                                             bool mergeBlockData) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
-  int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
+  int64_t             tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader);
 
   STSRow*    pTSRow = NULL;
   SRowMerger merge = {0};
+  TSDBROW    fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
 
-  TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
-  tRowMergerInit(&merge, &fRow, pReader->pSchema);
-  doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
+  // only last block exists
+  if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
+    if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) {
+      return TSDB_CODE_SUCCESS;
+    } else {
+      tRowMergerInit(&merge, &fRow, pReader->pSchema);
+      doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
 
-  // merge with block data if ts == key
-  if (mergeBlockData && (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
-    doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+      // merge with block data if ts == key
+      if (mergeBlockData && (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
+        doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+      }
+
+      int32_t code = tRowMergerGetRow(&merge, &pTSRow);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+
+      taosMemoryFree(pTSRow);
+      tRowMergerClear(&merge);
+    }
+  } else {  // not merge block data
+    tRowMergerInit(&merge, &fRow, pReader->pSchema);
+    doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
+
+    // merge with block data if ts == key
+    if (mergeBlockData && (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
+      doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
+    }
+
+    int32_t code = tRowMergerGetRow(&merge, &pTSRow);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+
+    taosMemoryFree(pTSRow);
+    tRowMergerClear(&merge);
   }
 
-  int32_t code = tRowMergerGetRow(&merge, &pTSRow);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
-
-  doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
-
-  taosMemoryFree(pTSRow);
-  tRowMergerClear(&merge);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1858,21 +1916,6 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
 
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
-static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
-  while (1) {
-    bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
-    if (!hasVal) {
-      return false;
-    }
-
-    TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
-    TSDBKEY k = TSDBROW_KEY(&row);
-    if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) {
-      return true;
-    }
-  }
-}
-
 static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
   // the last block reader has been initialized for this table.
   if (pLBlockReader->uid == pScanInfo->uid) {
@@ -1906,8 +1949,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan
 
 static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) {
   TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
-  TSDBKEY key = TSDBROW_KEY(&row);
-  return key.ts;
+  return TSDBROW_TS(&row);
 }
 
 static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; }

From 74260ebd70b024362bd05095852cf52be8ff7c32 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 7 Sep 2022 13:20:43 +0800
Subject: [PATCH 51/66] fix(query): add check.

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index a721cfdbfc..a01a817087 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -309,6 +309,10 @@ static void findNextValidRow(SLDataIter *pIter) {
   // mostly we only need to find the start position for a given table
   if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && pBlockData->aUid != NULL) {
     i = binarySearchForStartRowIndex((uint64_t*)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward);
+    if (i == -1) {
+      pIter->iRow = -1;
+      return;
+    }
   }
 
   for (; i < pBlockData->nRow && i >= 0; i += step) {

From 36414f33c9a8d19ab4211f478d195760f534e3ee Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Wed, 7 Sep 2022 13:38:29 +0800
Subject: [PATCH 52/66] feat: update taostools 023a01e for3.0 (#16709)

* feat: update taos-tools f169c0f for 3.0

* feat: update taos-tools a4d9b92 for 3.0

* feat: update taostools 0fa3a66 for 3.0

* feat: udpate taos-tools 023a01e for 3.0

* feat: update taos-tools 7d5c1c0 for 3.0
---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index b905e30489..adefacc6af 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG a4d9b92
+        GIT_TAG 7d5c1c0
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 1f5d9a38e712e430d6e4efde87480660083441db Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao@163.com>
Date: Wed, 7 Sep 2022 13:59:25 +0800
Subject: [PATCH 53/66] feat(stream):change single interval disc buff and
 interval delete

---
 include/libs/stream/tstream.h                 |   2 +
 source/libs/executor/inc/executorimpl.h       |  10 +-
 source/libs/executor/src/executorimpl.c       | 107 ++++-
 source/libs/executor/src/groupoperator.c      | 256 ++++++-----
 source/libs/executor/src/scanoperator.c       | 103 ++++-
 source/libs/executor/src/timewindowoperator.c | 190 +++++---
 source/libs/stream/src/streamState.c          |  23 +
 tests/script/tsim/stream/basic1.sim           |   4 +-
 tests/script/tsim/stream/deleteInterval.sim   | 419 ++++++++++++++++++
 9 files changed, 914 insertions(+), 200 deletions(-)
 create mode 100644 tests/script/tsim/stream/deleteInterval.sim

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 2c27509008..afd8de6b1c 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -554,6 +554,8 @@ typedef struct {
 int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
 int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
 int32_t streamStateDel(SStreamState* pState, const SWinKey* key);
+int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
 void    streamFreeVal(void* val);
 
 SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key);
diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index a9826e0018..fc8f942015 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -411,7 +411,7 @@ typedef enum EStreamScanMode {
   STREAM_SCAN_FROM_READERHANDLE = 1,
   STREAM_SCAN_FROM_RES,
   STREAM_SCAN_FROM_UPDATERES,
-  STREAM_SCAN_FROM_DELETERES,
+  STREAM_SCAN_FROM_DELETE_DATA,
   STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
   STREAM_SCAN_FROM_DATAREADER_RANGE,
 } EStreamScanMode;
@@ -794,6 +794,7 @@ typedef struct SStreamPartitionOperatorInfo {
   void*                 parIte;
   SSDataBlock*          pInputDataBlock;
   int32_t               tsColIndex;
+  SSDataBlock*          pDelRes;
 } SStreamPartitionOperatorInfo;
 
 typedef struct STimeSliceOperatorInfo {
@@ -1108,6 +1109,13 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol
 bool    groupbyTbname(SNodeList* pGroupList);
 int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
 void*   destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
+int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup,
+                                   SGroupResInfo* pGroupResInfo);
+int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
+                           int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
+                           SExecTaskInfo* pTaskInfo);
+int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult);
+int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 205bcd58df..9862aebdf8 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -3938,7 +3938,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
                                        pTaskInfo, isStream);
 
-  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { 
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
     pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
     SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
@@ -4410,3 +4410,108 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF
 
   return code;
 }
+
+int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
+                     int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
+                     SExecTaskInfo* pTaskInfo) {
+  SWinKey key = {
+      .ts = win->skey,
+      .groupId = tableGroupId,
+  };
+  char*   value = NULL;
+  int32_t size = pAggSup->resultRowSize;
+  if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
+  *pResult = (SResultRow*)value;
+  ASSERT(*pResult);
+  // set time window for current result
+  (*pResult)->win = (*win);
+  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) {
+  streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
+  streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup,
+                                   SGroupResInfo* pGroupResInfo) {
+  SExprInfo*      pExprInfo = pSup->pExprInfo;
+  int32_t         numOfExprs = pSup->numOfExprs;
+  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
+  SqlFunctionCtx* pCtx = pSup->pCtx;
+
+  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
+
+  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
+    SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
+    int32_t     size = 0;
+    void*       pVal = NULL;
+    SWinKey     key = {
+            .ts = *(TSKEY*)pPos->key,
+            .groupId = pPos->groupId,
+    };
+    int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size);
+    ASSERT(code == 0);
+    SResultRow* pRow = (SResultRow*)pVal;
+    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
+    // no results, continue to check the next one
+    if (pRow->numOfRows == 0) {
+      pGroupResInfo->index += 1;
+      releaseOutputBuf(pTaskInfo, &key, pRow);
+      continue;
+    }
+
+    if (pBlock->info.groupId == 0) {
+      pBlock->info.groupId = pPos->groupId;
+    } else {
+      // current value belongs to different group, it can't be packed into one datablock
+      if (pBlock->info.groupId != pPos->groupId) {
+        releaseOutputBuf(pTaskInfo, &key, pRow);
+        break;
+      }
+    }
+
+    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
+      ASSERT(pBlock->info.rows > 0);
+      releaseOutputBuf(pTaskInfo, &key, pRow);
+      break;
+    }
+
+    pGroupResInfo->index += 1;
+
+    for (int32_t j = 0; j < numOfExprs; ++j) {
+      int32_t slotId = pExprInfo[j].base.resSchema.slotId;
+
+      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
+      if (pCtx[j].fpSet.finalize) {
+        int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
+        if (TAOS_FAILED(code)) {
+          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
+          T_LONG_JMP(pTaskInfo->env, code);
+        }
+      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
+        // do nothing, todo refactor
+      } else {
+        // expand the result into multiple rows. E.g., _wstart, top(k, 20)
+        // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
+        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
+        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
+        for (int32_t k = 0; k < pRow->numOfRows; ++k) {
+          colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
+        }
+      }
+    }
+    releaseOutputBuf(pTaskInfo, &key, pRow);
+    pBlock->info.rows += pRow->numOfRows;
+  }
+  blockDataUpdateTsWindow(pBlock, 0);
+  return TSDB_CODE_SUCCESS;
+}
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 599edb0722..0c35ed5335 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -13,26 +13,26 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
 #include "function.h"
+#include "os.h"
 #include "tname.h"
 
 #include "tdatablock.h"
 #include "tmsg.h"
 
+#include "executorInt.h"
 #include "executorimpl.h"
 #include "tcompare.h"
 #include "thash.h"
 #include "ttypes.h"
-#include "executorInt.h"
 
 static void*    getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len);
 static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity);
-static int32_t  setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes,
-                                        uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup);
+static int32_t  setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData,
+                                        int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup);
 
 static void freeGroupKey(void* param) {
-  SGroupKeys* pKey = (SGroupKeys*) param;
+  SGroupKeys* pKey = (SGroupKeys*)param;
   taosMemoryFree(pKey->pData);
 }
 
@@ -62,13 +62,13 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char**
   int32_t numOfGroupCols = taosArrayGetSize(pGroupColList);
   for (int32_t i = 0; i < numOfGroupCols; ++i) {
     SColumn* pCol = taosArrayGet(pGroupColList, i);
-    (*keyLen) += pCol->bytes; // actual data + null_flag
+    (*keyLen) += pCol->bytes;  // actual data + null_flag
 
     SGroupKeys key = {0};
-    key.bytes  = pCol->bytes;
-    key.type   = pCol->type;
+    key.bytes = pCol->bytes;
+    key.type = pCol->type;
     key.isNull = false;
-    key.pData  = taosMemoryCalloc(1, pCol->bytes);
+    key.pData = taosMemoryCalloc(1, pCol->bytes);
     if (key.pData == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
@@ -87,7 +87,8 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char**
   return TSDB_CODE_SUCCESS;
 }
 
-static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) {
+static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex,
+                            int32_t numOfGroupCols) {
   SColumnDataAgg* pColAgg = NULL;
   for (int32_t i = 0; i < numOfGroupCols; ++i) {
     SColumn*         pCol = taosArrayGet(pGroupCols, i);
@@ -112,7 +113,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo
     if (pkey->type == TSDB_DATA_TYPE_JSON) {
       int32_t dataLen = getJsonValueLen(val);
 
-      if (memcmp(pkey->pData, val, dataLen) == 0){
+      if (memcmp(pkey->pData, val, dataLen) == 0) {
         continue;
       } else {
         return false;
@@ -154,7 +155,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
       pkey->isNull = false;
       char* val = colDataGetData(pColInfoData, rowIndex);
       if (pkey->type == TSDB_DATA_TYPE_JSON) {
-        if(tTagIsJson(val)){
+        if (tTagIsJson(val)) {
           terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
           return;
         }
@@ -198,13 +199,13 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) {
     }
   }
 
-  return (int32_t) (pStart - (char*)pKey);
+  return (int32_t)(pStart - (char*)pKey);
 }
 
 // assign the group keys or user input constant values if required
 static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) {
   for (int32_t i = 0; i < numOfOutput; ++i) {
-    if (pCtx[i].functionId == -1) {       // select count(*),key from t group by key.
+    if (pCtx[i].functionId == -1) {  // select count(*),key from t group by key.
       SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]);
 
       SColumnInfoData* pColInfoData = pCtx[i].input.pData[0];
@@ -221,7 +222,7 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t
         } else {
           memcpy(dest, data, pColInfoData->info.bytes);
         }
-      } else { // it is a NULL value
+      } else {  // it is a NULL value
         pEntryInfo->isNullRes = 1;
       }
 
@@ -275,7 +276,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     }
 
     len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
-    int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
+    int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf,
+                                          len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
     if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
       T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
     }
@@ -291,9 +293,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
 
   if (num > 0) {
     len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
-    int32_t ret =
-        setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len,
-                                pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
+    int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf,
+                                          len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
     if (ret != TSDB_CODE_SUCCESS) {
       T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
     }
@@ -308,7 +309,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
   SGroupbyOperatorInfo* pInfo = pOperator->info;
 
   SSDataBlock* pRes = pInfo->binfo.pRes;
-  while(1) {
+  while (1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     doFilter(pInfo->pCondition, pRes, NULL);
 
@@ -323,7 +324,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
   }
 
   pOperator->resultInfo.totalRows += pRes->info.rows;
-  return (pRes->info.rows == 0)? NULL:pRes;
+  return (pRes->info.rows == 0) ? NULL : pRes;
 }
 
 static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
@@ -334,7 +335,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
   SGroupbyOperatorInfo* pInfo = pOperator->info;
-  SSDataBlock* pRes = pInfo->binfo.pRes;
+  SSDataBlock*          pRes = pInfo->binfo.pRes;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
     return buildGroupResultDataBlock(pOperator);
@@ -343,7 +344,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   int32_t order = TSDB_ORDER_ASC;
   int32_t scanFlag = MAIN_SCAN;
 
-  int64_t st = taosGetTimestampUs();
+  int64_t        st = taosGetTimestampUs();
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
   while (1) {
@@ -362,7 +363,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
 
     // there is an scalar expression that needs to be calculated right before apply the group aggregation.
     if (pInfo->scalarSup.pExprInfo != NULL) {
-      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL);
+      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
+                                              pInfo->scalarSup.numOfExprs, NULL);
       if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
         T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
       }
@@ -403,8 +405,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
     goto _error;
   }
 
-  pInfo->pGroupCols      = pGroupColList;
-  pInfo->pCondition      = pCondition;
+  pInfo->pGroupCols = pGroupColList;
+  pInfo->pCondition = pCondition;
 
   int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
   if (code != TSDB_CODE_SUCCESS) {
@@ -425,14 +427,15 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
   initBasicInfo(&pInfo->binfo, pResultBlock);
   initResultRowInfo(&pInfo->binfo.resultRowInfo);
 
-  pOperator->name         = "GroupbyAggOperator";
-  pOperator->blocking     = true;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->name = "GroupbyAggOperator";
+  pOperator->blocking = true;
+  pOperator->status = OP_NOT_OPENED;
   // pOperator->operatorType = OP_Groupby;
-  pOperator->info         = pInfo;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->info = pInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL,
+                                         destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
@@ -440,7 +443,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
 
   return pOperator;
 
-  _error:
+_error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   destroyGroupOperatorInfo(pInfo);
   taosMemoryFreeClear(pOperator);
@@ -448,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
 }
 
 static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
-//  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  //  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
   SPartitionOperatorInfo* pInfo = pOperator->info;
 
@@ -457,7 +460,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
 
     SDataGroupInfo* pGroupInfo = NULL;
-    void *pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len);
+    void*           pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len);
 
     pGroupInfo->numOfRows += 1;
 
@@ -467,32 +470,32 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     }
 
     // number of rows
-    int32_t* rows = (int32_t*) pPage;
+    int32_t* rows = (int32_t*)pPage;
 
     size_t numOfCols = pOperator->exprSupp.numOfExprs;
-    for(int32_t i = 0; i < numOfCols; ++i) {
+    for (int32_t i = 0; i < numOfCols; ++i) {
       SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i];
-      int32_t slotId = pExpr->base.pParam[0].pCol->slotId;
+      int32_t    slotId = pExpr->base.pParam[0].pCol->slotId;
 
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
 
       int32_t bytes = pColInfoData->info.bytes;
       int32_t startOffset = pInfo->columnOffset[i];
 
-      int32_t* columnLen  = NULL;
+      int32_t* columnLen = NULL;
       int32_t  contentLen = 0;
 
       if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
         int32_t* offset = (int32_t*)((char*)pPage + startOffset);
-        columnLen       = (int32_t*) ((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity);
-        char*    data   = (char*)((char*) columnLen + sizeof(int32_t));
+        columnLen = (int32_t*)((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity);
+        char* data = (char*)((char*)columnLen + sizeof(int32_t));
 
         if (colDataIsNull_s(pColInfoData, j)) {
           offset[(*rows)] = -1;
           contentLen = 0;
-        } else if(pColInfoData->info.type == TSDB_DATA_TYPE_JSON){
+        } else if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
           offset[*rows] = (*columnLen);
-          char* src = colDataGetData(pColInfoData, j);
+          char*   src = colDataGetData(pColInfoData, j);
           int32_t dataLen = getJsonValueLen(src);
 
           memcpy(data + (*columnLen), src, dataLen);
@@ -511,8 +514,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
         }
       } else {
         char* bitmap = (char*)pPage + startOffset;
-        columnLen    = (int32_t*) ((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity));
-        char* data   = (char*) columnLen + sizeof(int32_t);
+        columnLen = (int32_t*)((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity));
+        char* data = (char*)columnLen + sizeof(int32_t);
 
         bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j);
         if (isNull) {
@@ -539,7 +542,7 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
   SDataGroupInfo* p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len);
 
   void* pPage = NULL;
-  if (p == NULL) { // it is a new group
+  if (p == NULL) {  // it is a new group
     SDataGroupInfo gi = {0};
     gi.pPageList = taosArrayInit(100, sizeof(int32_t));
     taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo));
@@ -550,12 +553,12 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
     pPage = getNewBufPage(pInfo->pBuf, &pageId);
     taosArrayPush(p->pPageList, &pageId);
 
-    *(int32_t *) pPage = 0;
+    *(int32_t*)pPage = 0;
   } else {
     int32_t* curId = taosArrayGetLast(p->pPageList);
     pPage = getBufPage(pInfo->pBuf, *curId);
 
-    int32_t *rows = (int32_t*) pPage;
+    int32_t* rows = (int32_t*)pPage;
     if (*rows >= pInfo->rowCapacity) {
       // release buffer
       releaseBufPage(pInfo->pBuf, pPage);
@@ -585,17 +588,18 @@ uint64_t calcGroupId(char* pData, int32_t len) {
 }
 
 int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) {
-  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+  size_t   numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t));
 
-  offset[0] = sizeof(int32_t) + sizeof(uint64_t);  // the number of rows in current page, ref to SSDataBlock paged serialization format
+  offset[0] = sizeof(int32_t) +
+              sizeof(uint64_t);  // the number of rows in current page, ref to SSDataBlock paged serialization format
 
-  for(int32_t i = 0; i < numOfCols - 1; ++i) {
+  for (int32_t i = 0; i < numOfCols - 1; ++i) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
 
     int32_t bytes = pColInfoData->info.bytes;
     int32_t payloadLen = bytes * rowCapacity;
-    
+
     if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
       // offset segment + content length + payload
       offset[i + 1] = rowCapacity * sizeof(int32_t) + sizeof(int32_t) + payloadLen + offset[i];
@@ -609,9 +613,9 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) {
 }
 
 static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) {
-  void *ite = NULL;
-  while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) {
-    taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList);
+  void* ite = NULL;
+  while ((ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL) {
+    taosArrayDestroy(((SDataGroupInfo*)ite)->pPageList);
   }
   taosArrayClear(pInfo->sortedGroupArray);
   clearDiskbasedBuf(pInfo->pBuf);
@@ -626,13 +630,14 @@ static int compareDataGroupInfo(const void* group1, const void* group2) {
     return 0;
   }
 
-  return (pGroupInfo1->groupId < pGroupInfo2->groupId)? -1:1;
+  return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1;
 }
 
 static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
   SPartitionOperatorInfo* pInfo = pOperator->info;
 
-  SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL;
+  SDataGroupInfo* pGroupInfo =
+      (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL;
   if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) {
     // try next group data
     ++pInfo->groupIndex;
@@ -647,7 +652,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
   }
 
   int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex);
-  void* page = getBufPage(pInfo->pBuf, *pageId);
+  void*    page = getBufPage(pInfo->pBuf, *pageId);
 
   blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity);
   blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity);
@@ -670,14 +675,14 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
   SPartitionOperatorInfo* pInfo = pOperator->info;
-  SSDataBlock* pRes = pInfo->binfo.pRes;
+  SSDataBlock*            pRes = pInfo->binfo.pRes;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
     blockDataCleanup(pRes);
     return buildPartitionResult(pOperator);
   }
 
-  int64_t st = taosGetTimestampUs();
+  int64_t        st = taosGetTimestampUs();
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
   while (1) {
@@ -688,7 +693,8 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
 
     // there is an scalar expression that needs to be calculated right before apply the group aggregation.
     if (pInfo->scalarSup.pExprInfo != NULL) {
-      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL);
+      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
+                                              pInfo->scalarSup.numOfExprs, NULL);
       if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
         T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
       }
@@ -727,7 +733,7 @@ static void destroyPartitionOperatorInfo(void* param) {
   cleanupBasicInfo(&pInfo->binfo);
   taosArrayDestroy(pInfo->pGroupCols);
 
-  for(int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++){
+  for (int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++) {
     SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->pGroupColVals, i);
     taosMemoryFree(key.pData);
   }
@@ -743,24 +749,25 @@ static void destroyPartitionOperatorInfo(void* param) {
   taosMemoryFreeClear(param);
 }
 
-SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
+                                           SExecTaskInfo* pTaskInfo) {
   SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo));
-  SOperatorInfo*        pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     goto _error;
   }
 
   SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc);
 
-  int32_t numOfCols = 0;
+  int32_t    numOfCols = 0;
   SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols);
 
   pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys);
 
   if (pPartNode->pExprs != NULL) {
-    int32_t num = 0;
+    int32_t    num = 0;
     SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num);
-    int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num);
+    int32_t    code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -772,7 +779,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
     goto _error;
   }
 
-  uint32_t defaultPgsz  = 0;
+  uint32_t defaultPgsz = 0;
   uint32_t defaultBufsz = 0;
   getBufferPgSize(pResBlock->info.rowSize, &defaultPgsz, &defaultBufsz);
 
@@ -794,15 +801,15 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
     goto _error;
   }
 
-  pOperator->name         = "PartitionOperator";
-  pOperator->blocking     = true;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->name = "PartitionOperator";
+  pOperator->blocking = true;
+  pOperator->status = OP_NOT_OPENED;
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION;
-  pInfo->binfo.pRes       = pResBlock;
-  pOperator->exprSupp.numOfExprs   = numOfCols;
-  pOperator->exprSupp.pExprInfo    = pExprInfo;
-  pOperator->info         = pInfo;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pInfo->binfo.pRes = pResBlock;
+  pOperator->exprSupp.numOfExprs = numOfCols;
+  pOperator->exprSupp.pExprInfo = pExprInfo;
+  pOperator->info = pInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo,
                                          NULL, NULL, NULL);
@@ -810,16 +817,16 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 
-  _error:
+_error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   taosMemoryFreeClear(pInfo);
   taosMemoryFreeClear(pOperator);
   return NULL;
 }
 
-int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes,
-                                uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) {
-  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData,
+                                int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) {
+  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
   SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo;
   SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
 
@@ -833,37 +840,36 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo,
 
 uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) {
   if (pExprSup->pExprInfo != NULL) {
-    int32_t code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    int32_t code =
+        projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
     if (code != TSDB_CODE_SUCCESS) {
       qError("calaculate group id error, code:%d", code);
     }
   }
   recordNewGroupKeys(pParSup->pGroupCols, pParSup->pGroupColVals, pBlock, rowId);
-  int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals);
+  int32_t  len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals);
   uint64_t groupId = calcGroupId(pParSup->keyBuf, len);
   return groupId;
 }
 
-static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) {
-  return pInfo->parIte != NULL;
-}
+static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; }
 
 static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
   SStreamPartitionOperatorInfo* pInfo = pOperator->info;
-  SSDataBlock* pDest = pInfo->binfo.pRes;
+  SSDataBlock*                  pDest = pInfo->binfo.pRes;
   ASSERT(hasRemainPartion(pInfo));
   SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte;
   blockDataCleanup(pDest);
-  int32_t rows = taosArrayGetSize(pParInfo->rowIds);
+  int32_t      rows = taosArrayGetSize(pParInfo->rowIds);
   SSDataBlock* pSrc = pInfo->pInputDataBlock;
   for (int32_t i = 0; i < rows; i++) {
     int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i);
     for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) {
-      int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId;
+      int32_t          slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId;
       SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId);
       SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j);
-      bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL);
-      char* pSrcData = colDataGetData(pSrcCol, rowIndex);
+      bool             isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL);
+      char*            pSrcData = colDataGetData(pSrcCol, rowIndex);
       colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull);
     }
     pDest->info.rows++;
@@ -881,9 +887,9 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat
   pInfo->pInputDataBlock = pBlock;
   for (int32_t i = 0; i < pBlock->info.rows; ++i) {
     recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i);
-    int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals);
-    SPartitionDataInfo* pParData = 
-        (SPartitionDataInfo*) taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen);
+    int32_t             keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals);
+    SPartitionDataInfo* pParData =
+        (SPartitionDataInfo*)taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen);
     if (pParData) {
       taosArrayPush(pParData->rowIds, &i);
     } else {
@@ -891,8 +897,7 @@ static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDat
       newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen);
       newParData.rowIds = taosArrayInit(64, sizeof(int32_t));
       taosArrayPush(newParData.rowIds, &i);
-      taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData,
-          sizeof(SPartitionDataInfo));
+      taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, sizeof(SPartitionDataInfo));
     }
   }
 }
@@ -902,13 +907,13 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
     return NULL;
   }
 
-  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*                pTaskInfo = pOperator->pTaskInfo;
   SStreamPartitionOperatorInfo* pInfo = pOperator->info;
   if (hasRemainPartion(pInfo)) {
     return buildStreamPartitionResult(pOperator);
   }
 
-  int64_t st = taosGetTimestampUs();
+  int64_t        st = taosGetTimestampUs();
   SOperatorInfo* downstream = pOperator->pDownstream[0];
   {
     pInfo->pInputDataBlock = NULL;
@@ -924,14 +929,18 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
       case STREAM_INVALID:
         pInfo->binfo.pRes->info.type = pBlock->info.type;
         break;
+      case STREAM_DELETE_DATA: {
+        copyDataBlock(pInfo->pDelRes, pBlock);
+        pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
+      } break;
       default:
         return pBlock;
     }
 
     // there is an scalar expression that needs to be calculated right before apply the group aggregation.
     if (pInfo->scalarSup.pExprInfo != NULL) {
-      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock,
-          pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL);
+      pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
+                                              pInfo->scalarSup.numOfExprs, NULL);
       if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
         longjmp(pTaskInfo->env, pTaskInfo->code);
       }
@@ -940,7 +949,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) {
     doStreamHashPartitionImpl(pInfo, pBlock);
   }
   pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
-  
+
   pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL);
   return buildStreamPartitionResult(pOperator);
 }
@@ -950,7 +959,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
   cleanupBasicInfo(&pInfo->binfo);
   taosArrayDestroy(pInfo->partitionSup.pGroupCols);
 
-  for(int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++){
+  for (int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++) {
     SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i);
     taosMemoryFree(key.pData);
   }
@@ -958,6 +967,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
 
   taosMemoryFree(pInfo->partitionSup.keyBuf);
   cleanupExprSupp(&pInfo->scalarSup);
+  blockDataDestroy(pInfo->pDelRes);
   taosMemoryFreeClear(param);
 }
 
@@ -970,7 +980,8 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup
   pScanInfo->pPartScalarSup = pExpr;
 }
 
-SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
+                                                 SExecTaskInfo* pTaskInfo) {
   SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
   SOperatorInfo*                pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -980,7 +991,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
   pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys);
 
   if (pPartNode->pExprs != NULL) {
-    int32_t num = 0;
+    int32_t    num = 0;
     SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num);
     code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num);
     if (code != TSDB_CODE_SUCCESS) {
@@ -989,7 +1000,8 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
   }
 
   int32_t keyLen = 0;
-  code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols);
+  code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf,
+                           pInfo->partitionSup.pGroupCols);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -1000,35 +1012,35 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
     goto _error;
   }
   blockDataEnsureCapacity(pResBlock, 4096);
-  pInfo->binfo.pRes       = pResBlock;
-  pInfo->parIte           = NULL;
-  pInfo->pInputDataBlock  = NULL;
+  pInfo->binfo.pRes = pResBlock;
+  pInfo->parIte = NULL;
+  pInfo->pInputDataBlock = NULL;
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
-  pInfo->pPartitions      = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
-  pInfo->tsColIndex       = 0;
+  pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
+  pInfo->tsColIndex = 0;
+  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
 
-  int32_t numOfCols = 0;
+  int32_t    numOfCols = 0;
   SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols);
 
-  pOperator->name         = "StreamPartitionOperator";
-  pOperator->blocking     = false;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->name = "StreamPartitionOperator";
+  pOperator->blocking = false;
+  pOperator->status = OP_NOT_OPENED;
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION;
-  pOperator->exprSupp.numOfExprs   = numOfCols;
-  pOperator->exprSupp.pExprInfo    = pExprInfo;
-  pOperator->info         = pInfo;
-  pOperator->pTaskInfo    = pTaskInfo;
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, destroyStreamPartitionOperatorInfo,
-                                         NULL, NULL, NULL);
+  pOperator->exprSupp.numOfExprs = numOfCols;
+  pOperator->exprSupp.pExprInfo = pExprInfo;
+  pOperator->info = pInfo;
+  pOperator->pTaskInfo = pTaskInfo;
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL,
+                                         destroyStreamPartitionOperatorInfo, NULL, NULL, NULL);
 
   initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup);
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 
-  _error:
+_error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
-  taosMemoryFreeClear(pInfo);
+  destroyStreamPartitionOperatorInfo(pInfo);
   taosMemoryFreeClear(pOperator);
   return NULL;
 }
-
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index c099c1c24c..ad9cd1ffe7 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1057,24 +1057,24 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
   return true;
 }
 
-static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo,
-                                    int32_t* pRowIndex, bool hasGroup) {
+static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval,
+                                    SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
   SResultRowInfo dumyInfo;
   dumyInfo.cur.pageId = -1;
-  STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
+  STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
   STimeWindow endWin = win;
   STimeWindow preWin = win;
   while (1) {
     if (hasGroup) {
       (*pRowIndex) += 1;
     } else {
-      (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL,
-                                               TSDB_ORDER_ASC);
+      (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey,
+                                               NULL, TSDB_ORDER_ASC);
     }
     do {
       preWin = endWin;
       getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC);
-    } while (tsCol[(*pRowIndex) - 1] >= endWin.skey);
+    } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey);
     endWin = preWin;
     if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) {
       win.ekey = endWin.ekey;
@@ -1102,6 +1102,11 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
       return NULL;
     }
 
+    doFilter(pInfo->pCondition, pResult, NULL);
+    if (pResult->info.rows == 0) {
+      continue;
+    }
+
     if (pInfo->partitionSup.needCalc) {
       SSDataBlock* tmpBlock = createOneDataBlock(pResult, true);
       blockDataCleanup(pResult);
@@ -1188,13 +1193,15 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
     return code;
   }
 
-  SColumnInfoData* pSrcTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
   SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
   uint64_t*        srcUidData = (uint64_t*)pSrcUidCol->pData;
   SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   uint64_t*        srcGp = (uint64_t*)pSrcGpCol->pData;
-  ASSERT(pSrcTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
-  TSKEY*           tsCol = (TSKEY*)pSrcTsCol->pData;
+  ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+  TSKEY*           srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
+  TSKEY*           srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
   SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
   SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
   SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
@@ -1204,12 +1211,13 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
   int64_t          version = pSrcBlock->info.version - 1;
   for (int32_t i = 0; i < rows;) {
     uint64_t srcUid = srcUidData[i];
-    uint64_t groupId = getGroupIdByData(pInfo, srcUid, tsCol[i], version);
+    uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version);
     uint64_t srcGpId = srcGp[i];
-    TSKEY    calStartTs = tsCol[i];
+    TSKEY    calStartTs = srcStartTsCol[i];
     colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
-    STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i, pInfo->partitionSup.needCalc);
-    TSKEY       calEndTs = tsCol[i - 1];
+    STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i,
+                                       pInfo->partitionSup.needCalc);
+    TSKEY       calEndTs = srcStartTsCol[i - 1];
     colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
     colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
     colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
@@ -1229,11 +1237,49 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
+  if (pSrcBlock->info.rows == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+  blockDataCleanup(pDestBlock);
+  int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+  ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3);
+  SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
+  SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
+  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
+  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
+
+  SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
+  SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+  SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
+  int32_t          dummy = 0;
+  int64_t          version = pSrcBlock->info.version - 1;
+  for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
+    uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version);
+    colDataAppend(pDestStartCol, i, (const char*)(startData + i), false);
+    colDataAppend(pDestEndCol, i, (const char*)(endData + i), false);
+    colDataAppendNULL(pDestUidCol, i);
+    colDataAppend(pDestGpCol, i, (const char*)&groupId, false);
+    colDataAppendNULL(pDestCalStartTsCol, i);
+    colDataAppendNULL(pDestCalEndTsCol, i);
+    pDestBlock->info.rows++;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
   int32_t code = TSDB_CODE_SUCCESS;
   if (isIntervalWindow(pInfo)) {
     code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock);
-  } else {
+  } else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) {
     code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock);
   }
   pDestBlock->info.type = STREAM_CLEAR;
@@ -1510,14 +1556,23 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
       } break;
       case STREAM_DELETE_DATA: {
-        pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
-        pInfo->updateResIndex = 0;
-        generateScanRange(pInfo, pBlock, pInfo->pUpdateRes);
-        prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
-        copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
-        pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
-        pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
-        return pInfo->pDeleteDataRes;
+        printDataBlock(pBlock, "stream scan delete recv");
+        if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) {
+          generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes);
+          pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;
+          printDataBlock(pBlock, "stream scan delete result");
+          return pInfo->pDeleteDataRes;
+        } else {
+          pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
+          pInfo->updateResIndex = 0;
+          generateScanRange(pInfo, pBlock, pInfo->pUpdateRes);
+          prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+          copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
+          pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
+          pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
+          printDataBlock(pBlock, "stream scan delete data");
+          return pInfo->pDeleteDataRes;
+        }
       } break;
       default:
         break;
@@ -1532,7 +1587,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
         return pInfo->pRes;
       } break;
-      case STREAM_SCAN_FROM_DELETERES: {
+      case STREAM_SCAN_FROM_DELETE_DATA: {
         generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
         prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
@@ -1646,7 +1701,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
           pInfo->scanMode = STREAM_SCAN_FROM_RES;
           return pInfo->pUpdateDataRes;
         } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) {
-          pInfo->scanMode = STREAM_SCAN_FROM_DELETERES;
+          pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA;
         }
       }
     }
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index d773f8a629..f158b24b58 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -955,8 +955,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
 
   STimeWindow win =
       getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
-  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
-                                numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
+  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
+                                       pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
   if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
     T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
@@ -983,7 +983,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
 
   updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
   doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
-                    numOfOutput);
+                   numOfOutput);
 
   doCloseWindow(pResultRowInfo, pInfo, pResult);
 
@@ -1406,20 +1406,25 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock,
                                    SHashObj* pUpdatedMap) {
   SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
   TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
+  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  TSKEY*           tsEnds = (TSKEY*)pEndCol->pData;
   SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
   for (int32_t i = 0; i < pBlock->info.rows; i++) {
     SResultRowInfo dumyInfo;
     dumyInfo.cur.pageId = -1;
     STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC);
-    doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
-    SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
-    if (pDelWins) {
-      taosArrayPush(pDelWins, &winRes);
-    }
-    if (pUpdatedMap) {
-      taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
-    }
+    do {
+      doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]);
+      SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]};
+      if (pDelWins) {
+        taosArrayPush(pDelWins, &winRes);
+      }
+      if (pUpdatedMap) {
+        taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
+      }
+      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
+    } while (win.skey < tsEnds[i]);
   }
 }
 
@@ -2775,7 +2780,7 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr
       updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true);
       compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData);
     }
-    if (num > 1 && pUpdatedMap) {
+    if (num > 0 && pUpdatedMap) {
       saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap);
       setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur);
     }
@@ -2807,15 +2812,14 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {
 
 static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }
 
-static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
-                           SHashObj* pUpdatedMap) {
+static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
+                              SHashObj* pUpdatedMap) {
   SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info;
   SResultRowInfo*                   pResultRowInfo = &(pInfo->binfo.resultRowInfo);
   SExecTaskInfo*                    pTaskInfo = pOperatorInfo->pTaskInfo;
   SExprSupp*                        pSup = &pOperatorInfo->exprSupp;
   int32_t                           numOfOutput = pSup->numOfExprs;
   int32_t                           step = 1;
-  bool                              ascScan = true;
   TSKEY*                            tsCols = NULL;
   SResultRow*                       pResult = NULL;
   int32_t                           forwardRows = 0;
@@ -2824,7 +2828,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
   SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
   tsCols = (int64_t*)pColDataInfo->pData;
 
-  int32_t     startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
+  int32_t     startPos = 0;
   TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
   STimeWindow nextWin = {0};
   if (IS_FINAL_OP(pInfo)) {
@@ -3165,7 +3169,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
     }
     setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true);
-    doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
+    doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
     if (IS_FINAL_OP(pInfo)) {
       int32_t chIndex = getChildIndex(pBlock);
       int32_t size = taosArrayGetSize(pInfo->pChildren);
@@ -3183,7 +3187,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       SOperatorInfo*                    pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);
       SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
       setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
-      doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
+      doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL);
     }
   }
 
@@ -5468,25 +5472,24 @@ _error:
   return NULL;
 }
 
-static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
-    SSDataBlock* pBlock, int32_t scanFlag, SHashObj* pUpdatedMap) {
+static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
+                                    int32_t scanFlag, SHashObj* pUpdatedMap) {
   SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;
 
   SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
   SExprSupp*     pSup = &pOperatorInfo->exprSupp;
 
-  int32_t     startPos = 0;
-  int32_t     numOfOutput = pSup->numOfExprs;
+  int32_t          startPos = 0;
+  int32_t          numOfOutput = pSup->numOfExprs;
   SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
-  TSKEY*      tsCols = (TSKEY*)pColDataInfo->pData;
-  uint64_t    tableGroupId = pBlock->info.groupId;
-  bool        ascScan = true;
-  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
-  SResultRow* pResult = NULL;
+  TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
+  uint64_t         tableGroupId = pBlock->info.groupId;
+  bool             ascScan = true;
+  TSKEY            ts = getStartTsKey(&pBlock->info.window, tsCols);
+  SResultRow*      pResult = NULL;
 
-  STimeWindow win =
-      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
-  int32_t ret = TSDB_CODE_SUCCESS;
+  STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
+  int32_t     ret = TSDB_CODE_SUCCESS;
   if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
       inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
     ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
@@ -5547,11 +5550,88 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo
   }
 }
 
+static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId,
+                                     SHashObj* pUpdatedMap) {
+  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;
+
+  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);
+  SExecTaskInfo*  pTaskInfo = pOperatorInfo->pTaskInfo;
+  SExprSupp*      pSup = &pOperatorInfo->exprSupp;
+  int32_t         numOfOutput = pSup->numOfExprs;
+  int32_t         step = 1;
+  TSKEY*          tsCols = NULL;
+  SResultRow*     pResult = NULL;
+  int32_t         forwardRows = 0;
+  int32_t         aa = 4;
+
+  ASSERT(pSDataBlock->pDataBlock != NULL);
+  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
+  tsCols = (int64_t*)pColDataInfo->pData;
+
+  int32_t     startPos = 0;
+  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
+  STimeWindow nextWin =
+      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
+  while (1) {
+    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
+    if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
+      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
+      if (startPos < 0) {
+        break;
+      }
+      continue;
+    }
+
+    int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset,
+                                &pInfo->aggSup, pTaskInfo);
+    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
+      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+
+    forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
+                                           TSDB_ORDER_ASC);
+    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
+      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
+    }
+    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
+    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
+                     pSDataBlock->info.rows, numOfOutput);
+    SWinKey key = {
+        .ts = nextWin.skey,
+        .groupId = tableGroupId,
+    };
+    saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize);
+    releaseOutputBuf(pTaskInfo, &key, pResult);
+    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
+    ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0);
+    startPos =
+        getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
+    if (startPos < 0) {
+      break;
+    }
+  }
+}
+
+void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) {
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  // set output datablock version
+  pBlock->info.version = pTaskInfo->version;
+
+  blockDataCleanup(pBlock);
+  if (!hasRemainResults(pGroupResInfo)) {
+    return;
+  }
+
+  // clear the existed group id
+  pBlock->info.groupId = 0;
+  buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo);
+}
+
 static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   SStreamIntervalOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
-  int64_t                   maxTs = INT64_MIN;
-  SExprSupp* pSup = &pOperator->exprSupp;
+  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
+  int64_t                      maxTs = INT64_MIN;
+  SExprSupp*                   pSup = &pOperator->exprSupp;
 
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -5622,6 +5702,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     maxTs = TMAX(maxTs, pBlock->info.window.ekey);
     doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
+    // new disc buf
+    // doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);
   }
   pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
 
@@ -5664,6 +5746,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   }
   taosArraySort(pUpdated, resultrowComparAsc);
 
+  // new disc buf
   finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
@@ -5676,6 +5759,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   }
 
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+  // new disc buf
+  // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo);
   printDataBlock(pInfo->binfo.pRes, "single interval");
   return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
 }
@@ -5697,25 +5782,29 @@ void destroyStreamIntervalOperatorInfo(void* param) {
 SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                 SExecTaskInfo* pTaskInfo) {
   SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
-  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
     goto _error;
   }
   SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;
 
-  int32_t numOfCols = 0;
-  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
+  int32_t    numOfCols = 0;
+  SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
   ASSERT(numOfCols > 0);
   SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
-  SInterval interval = {.interval = pIntervalPhyNode->interval,
-                        .sliding = pIntervalPhyNode->sliding,
-                        .intervalUnit = pIntervalPhyNode->intervalUnit,
-                        .slidingUnit = pIntervalPhyNode->slidingUnit,
-                        .offset = pIntervalPhyNode->offset,
-                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, };
-  STimeWindowAggSupp twAggSupp = {.waterMark = pIntervalPhyNode->window.watermark,
-                                  .calTrigger = pIntervalPhyNode->window.triggerType,
-                                  .maxTs = INT64_MIN, };
+  SInterval    interval = {
+         .interval = pIntervalPhyNode->interval,
+         .sliding = pIntervalPhyNode->sliding,
+         .intervalUnit = pIntervalPhyNode->intervalUnit,
+         .slidingUnit = pIntervalPhyNode->slidingUnit,
+         .offset = pIntervalPhyNode->offset,
+         .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
+  };
+  STimeWindowAggSupp twAggSupp = {
+      .waterMark = pIntervalPhyNode->window.watermark,
+      .calTrigger = pIntervalPhyNode->window.triggerType,
+      .maxTs = INT64_MIN,
+  };
   ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
   pOperator->pTaskInfo = pTaskInfo;
   pInfo->interval = interval;
@@ -5732,11 +5821,11 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
     }
   }
 
-  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;;
+  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   SExprSupp* pSup = &pOperator->exprSupp;
-  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
+  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
+  int32_t    code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -5758,8 +5847,9 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
   pOperator->blocking = true;
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL,
-                                         destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
+  pOperator->fpSet =
+      createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo,
+                          aggEncodeResultRow, aggDecodeResultRow, NULL);
 
   initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
   code = appendDownstream(pOperator, &downstream, 1);
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index dfd6f012cc..5efdbb4679 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -112,6 +112,29 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key) {
   return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn);
 }
 
+int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
+  // todo refactor
+  int32_t size = *pVLen;
+  if (streamStateGet(pState, key, pVal, pVLen) == 0) {
+    return 0;
+  }
+  void* tmp = taosMemoryCalloc(1, size);
+  if (streamStatePut(pState, key, &tmp, size) == 0) {
+    taosMemoryFree(tmp);
+    int32_t code = streamStateGet(pState, key, pVal, pVLen);
+    ASSERT(code == 0);
+    return code;
+  }
+  taosMemoryFree(tmp);
+  return -1;
+}
+
+int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) {
+  // todo refactor
+  streamFreeVal(pVal);
+  return 0;
+}
+
 SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) {
   SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
   if (pCur == NULL) return NULL;
diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim
index 70ca1179b0..8942f7f702 100644
--- a/tests/script/tsim/stream/basic1.sim
+++ b/tests/script/tsim/stream/basic1.sim
@@ -5,7 +5,7 @@ sleep 50
 sql connect
 
 print =============== create database
-sql create database test vgroups 1
+sql create database test vgroups 1;
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1
@@ -13,7 +13,7 @@ endi
 
 print $data00 $data01 $data02
 
-sql use test
+sql use test;
 
 
 sql create table t1(ts timestamp, a int, b int , c int, d double);
diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim
new file mode 100644
index 0000000000..dfd0ddc9d3
--- /dev/null
+++ b/tests/script/tsim/stream/deleteInterval.sim
@@ -0,0 +1,419 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test  vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select  _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 0 then
+  print =====rows=$rows
+  goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop1
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,2,2,2,2.0);
+sql insert into t1 values(1648791213002,3,3,3,3.0);
+sql insert into t1 values(1648791213003,4,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop3
+endi
+
+if $data02 != 4 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop5
+endi
+
+if $data02 != 4 then
+  print =====data02=$data02
+  goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,2,2,2,2.0);
+sql insert into t1 values(1648791213006,3,3,3,3.0);
+sql insert into t1 values(1648791213007,4,4,4,4.0);
+
+sql insert into t1 values(1648791223000,1,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,3,3,3,3.0);
+sql insert into t1 values(1648791223003,4,4,4,4.0);
+
+sql insert into t1 values(1648791233000,1,1,1,1.0);
+sql insert into t1 values(1648791233001,2,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,4,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop6
+endi
+
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop6
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop6
+endi
+
+if $data12 != 4 then
+  print =====data12=$data12
+  goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2  vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select  _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 1 then
+  print =====rows=$rows
+  goto loop7
+endi
+
+sleep 200
+
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop8:
+sleep 200
+
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop8
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop8
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001;
+
+$loop_count = 0
+
+loop11:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop11
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop11
+endi
+
+if $data11 != 6 then
+  print =====data11=$data11
+  goto loop11
+endi
+
+if $data12 != 3 then
+  print =====data12=$data12
+  goto loop11
+endi
+
+sleep 200
+
+sql delete from st where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop12:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 1 then
+  print =====rows=$rows
+  goto loop12
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop12
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop12
+endi
+
+sql insert into t1 values(1648791213004,3,2,3,1.0);
+sql insert into t1 values(1648791213005,3,2,3,1.0);
+sql insert into t1 values(1648791213006,3,2,3,1.0);
+sql insert into t1 values(1648791223004,1,2,3,1.0);
+sql insert into t2 values(1648791213004,3,2,3,1.0);
+sql insert into t2 values(1648791213005,3,2,3,1.0);
+sql insert into t2 values(1648791213006,3,2,3,1.0);
+sql insert into t2 values(1648791223004,1,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006;
+
+$loop_count = 0
+
+loop13:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop13
+endi
+
+if $data01 != 4 then
+  print =====data01=$data01
+  goto loop13
+endi
+
+if $data02 != 3 then
+  print =====data02=$data02
+  goto loop13
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop13
+endi
+
+if $data12 != 1 then
+  print =====data12=$data12
+  goto loop13
+endi
+
+sql insert into t1 values(1648791223005,1,2,3,1.0);
+sql insert into t1 values(1648791223006,1,2,3,1.0);
+sql insert into t2 values(1648791223005,1,2,3,1.0);
+sql insert into t2 values(1648791223006,1,2,3,1.0);
+
+sql insert into t1 values(1648791233005,4,2,3,1.0);
+sql insert into t1 values(1648791233006,2,2,3,1.0);
+sql insert into t2 values(1648791233005,5,2,3,1.0);
+sql insert into t2 values(1648791233006,3,2,3,1.0);
+
+sleep 200
+
+sql delete from st where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop14:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop14
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop14
+endi
+
+if $data02 != NULL then
+  print =====data02=$data02
+  goto loop14
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop14
+endi
+
+if $data12 != 3 then
+  print =====data12=$data12
+  goto loop14
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file

From 69f717ec8b7df9da8599bec9e316f28603a35a03 Mon Sep 17 00:00:00 2001
From: Xuefeng Tan <1172915550@qq.com>
Date: Wed, 7 Sep 2022 15:11:34 +0800
Subject: [PATCH 54/66] enh(taosAdapter): enable experimental snapshot by
 default (#16715)

---
 cmake/taosadapter_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index f182beed33..6ff07fbb0d 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG abed566
+        GIT_TAG 22bdac5
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From d9363214b7b485ba6a30b85218e57846b1d36e5b Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 7 Sep 2022 15:29:04 +0800
Subject: [PATCH 55/66] refactor(sync): add syncHeartbeat

---
 include/common/tmsgdef.h                    |   2 +
 include/libs/sync/syncTools.h               |  46 ++++++
 source/libs/sync/src/syncMessage.c          | 153 ++++++++++++++++++++
 source/libs/sync/test/CMakeLists.txt        |  14 ++
 source/libs/sync/test/syncHeartbeatTest.cpp |  99 +++++++++++++
 5 files changed, 314 insertions(+)
 create mode 100644 source/libs/sync/test/syncHeartbeatTest.cpp

diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 006ba7f21b..33d01bc46d 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -261,6 +261,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES, "sync-append-entries", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES_BATCH, "sync-append-entries-batch", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES_REPLY, "sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_NOOP, "sync-noop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_UNKNOWN, "sync-unknown", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_COMMON_RESPONSE, "sync-common-response", NULL, NULL)
diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index 6c95c3c6d7..09e24477d7 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -444,6 +444,52 @@ void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
 void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
 void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
 
+// ---------------------------------------------
+typedef struct SyncHeartbeat {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  // private data
+  SyncTerm  term;
+  SyncIndex commitIndex;
+  SyncTerm  privateTerm;
+} SyncHeartbeat;
+
+SyncHeartbeat* syncHeartbeatBuild(int32_t vgId);
+void           syncHeartbeatDestroy(SyncHeartbeat* pMsg);
+void           syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen);
+void           syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg);
+char*          syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len);
+SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len);
+void           syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg);
+void           syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg);
+SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncHeartbeat2Json(const SyncHeartbeat* pMsg);
+char*          syncHeartbeat2Str(const SyncHeartbeat* pMsg);
+
+// for debug ----------------------
+void syncHeartbeatPrint(const SyncHeartbeat* pMsg);
+void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg);
+void syncHeartbeatLog(const SyncHeartbeat* pMsg);
+void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncHeartbeatReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  // private data
+  SyncTerm term;
+  SyncTerm privateTerm;
+  int64_t  startTime;
+} SyncHeartbeatReply;
+
 // ---------------------------------------------
 typedef struct SyncApplyMsg {
   uint32_t   bytes;
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index b42aba560f..a16f87059f 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -1992,6 +1992,159 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) {
   }
 }
 
+// ---- message process SyncHeartbeat----
+SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) {
+  uint32_t       bytes = sizeof(SyncHeartbeat);
+  SyncHeartbeat* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_APPEND_ENTRIES;
+  return pMsg;
+}
+
+void syncHeartbeatDestroy(SyncHeartbeat* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen) {
+  ASSERT(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg) {
+  memcpy(pMsg, buf, len);
+  ASSERT(len == pMsg->bytes);
+}
+
+char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  ASSERT(buf != NULL);
+  syncHeartbeatSerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len) {
+  uint32_t       bytes = *((uint32_t*)buf);
+  SyncHeartbeat* pMsg = taosMemoryMalloc(bytes);
+  ASSERT(pMsg != NULL);
+  syncHeartbeatDeserialize(buf, len, pMsg);
+  ASSERT(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncHeartbeatSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg) {
+  syncHeartbeatDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncHeartbeat* pMsg = syncHeartbeatDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  ASSERT(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg) {
+  char   u64buf[128] = {0};
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex);
+    cJSON_AddStringToObject(pRoot, "commitIndex", u64buf);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncHeartbeat", pRoot);
+  return pJson;
+}
+
+char* syncHeartbeat2Str(const SyncHeartbeat* pMsg) {
+  cJSON* pJson = syncHeartbeat2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+void syncHeartbeatPrint(const SyncHeartbeat* pMsg) {
+  char* serialized = syncHeartbeat2Str(pMsg);
+  printf("syncHeartbeatPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg) {
+  char* serialized = syncHeartbeat2Str(pMsg);
+  printf("syncHeartbeatPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncHeartbeatLog(const SyncHeartbeat* pMsg) {
+  char* serialized = syncHeartbeat2Str(pMsg);
+  sTrace("syncHeartbeatLog | len:%" PRIu64 " | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncHeartbeat2Str(pMsg);
+    sTrace("syncHeartbeatLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
 // ---- message process SyncApplyMsg----
 SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) {
   uint32_t      bytes = sizeof(SyncApplyMsg) + dataLen;
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 72845d0c1d..e2be1523c0 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -57,6 +57,7 @@ add_executable(syncLeaderTransferTest "")
 add_executable(syncReconfigFinishTest "")
 add_executable(syncRestoreFromSnapshot "")
 add_executable(syncRaftCfgIndexTest "")
+add_executable(syncHeartbeatTest "")
 
 
 target_sources(syncTest
@@ -295,6 +296,10 @@ target_sources(syncRaftCfgIndexTest
     PRIVATE
     "syncRaftCfgIndexTest.cpp"
 )
+target_sources(syncHeartbeatTest
+    PRIVATE
+    "syncHeartbeatTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -592,6 +597,11 @@ target_include_directories(syncRaftCfgIndexTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncHeartbeatTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -830,6 +840,10 @@ target_link_libraries(syncRaftCfgIndexTest
     sync
     gtest_main
 )
+target_link_libraries(syncHeartbeatTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp
new file mode 100644
index 0000000000..d910c828f1
--- /dev/null
+++ b/source/libs/sync/test/syncHeartbeatTest.cpp
@@ -0,0 +1,99 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncHeartbeat *createMsg() {
+  SyncHeartbeat *pMsg = syncHeartbeatBuild(789);
+  pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+  pMsg->srcId.vgId = 100;
+  pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+  pMsg->destId.vgId = 100;  
+  pMsg->term = 8;
+  pMsg->commitIndex = 33;
+  pMsg->privateTerm = 44;
+  return pMsg;
+}
+
+void test1() {
+  SyncHeartbeat *pMsg = createMsg();
+  syncHeartbeatLog2((char *)"test1:", pMsg);
+  syncHeartbeatDestroy(pMsg);
+}
+
+void test2() {
+  SyncHeartbeat *pMsg = createMsg();
+  uint32_t           len = pMsg->bytes;
+  char *             serialized = (char *)taosMemoryMalloc(len);
+  syncHeartbeatSerialize(pMsg, serialized, len);
+  SyncHeartbeat *pMsg2 = syncHeartbeatBuild(789);
+  syncHeartbeatDeserialize(serialized, len, pMsg2);
+  syncHeartbeatLog2((char *)"test2: syncHeartbeatSerialize -> syncHeartbeatDeserialize ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncHeartbeatDestroy(pMsg);
+  syncHeartbeatDestroy(pMsg2);
+}
+
+void test3() {
+  SyncHeartbeat *pMsg = createMsg();
+  uint32_t           len;
+  char *             serialized = syncHeartbeatSerialize2(pMsg, &len);
+  SyncHeartbeat *pMsg2 = syncHeartbeatDeserialize2(serialized, len);
+  syncHeartbeatLog2((char *)"test3: syncHeartbeatSerialize2 -> syncHeartbeatDeserialize2 ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncHeartbeatDestroy(pMsg);
+  syncHeartbeatDestroy(pMsg2);
+}
+
+void test4() {
+  SyncHeartbeat *pMsg = createMsg();
+  SRpcMsg            rpcMsg;
+  syncHeartbeat2RpcMsg(pMsg, &rpcMsg);
+  SyncHeartbeat *pMsg2 = (SyncHeartbeat *)taosMemoryMalloc(rpcMsg.contLen);
+  syncHeartbeatFromRpcMsg(&rpcMsg, pMsg2);
+  syncHeartbeatLog2((char *)"test4: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncHeartbeatDestroy(pMsg);
+  syncHeartbeatDestroy(pMsg2);
+}
+
+void test5() {
+  SyncHeartbeat *pMsg = createMsg();
+  SRpcMsg            rpcMsg;
+  syncHeartbeat2RpcMsg(pMsg, &rpcMsg);
+  SyncHeartbeat *pMsg2 =syncHeartbeatFromRpcMsg2(&rpcMsg);
+  syncHeartbeatLog2((char *)"test5: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg2 ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncHeartbeatDestroy(pMsg);
+  syncHeartbeatDestroy(pMsg2);
+}
+
+int main() {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  gRaftDetailLog = true;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+
+  return 0;
+}

From aa1bf1e3bc3b1068b3ef0ced8f73859847696dd2 Mon Sep 17 00:00:00 2001
From: dingbo <dingbo8128@163.com>
Date: Wed, 7 Sep 2022 16:32:53 +0800
Subject: [PATCH 56/66] docs/release 310

---
 docs/zh/28-releases/01-tdengine.md | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md
index e3e1463131..a6ec560d3c 100644
--- a/docs/zh/28-releases/01-tdengine.md
+++ b/docs/zh/28-releases/01-tdengine.md
@@ -6,11 +6,7 @@ description: TDengine 发布历史、Release Notes 及下载链接
 
 import Release from "/components/ReleaseV3";
 
-## 3.0.0.1
+## 3.0.1.0
 
-<Release type="tdengine" version="3.0.0.1" />
-
-<!-- ## 3.0.0.0
-
-<Release type="tdengine" version="3.0.0.0" /> -->
+<Release type="tdengine" version="3.0.1.0" />
 

From 99a102f51bf13f3320d79098b913e6f0dc1530fb Mon Sep 17 00:00:00 2001
From: dingbo <dingbo8128@163.com>
Date: Wed, 7 Sep 2022 17:04:16 +0800
Subject: [PATCH 57/66] docs: release tools

---
 docs/zh/28-releases/02-tools.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md
index 61129d74e5..c8a99cb40f 100644
--- a/docs/zh/28-releases/02-tools.md
+++ b/docs/zh/28-releases/02-tools.md
@@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接
 
 import Release from "/components/ReleaseV3";
 
+## 2.1.3
+
+<Release type="tools" version="2.1.3" />
+
 ## 2.1.2
 
 <Release type="tools" version="2.1.2" />
\ No newline at end of file

From a0a6c9e9b94dd3ebf47fde53fa7383fcecbec083 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 7 Sep 2022 17:51:04 +0800
Subject: [PATCH 58/66] fix(query): fix bug caused by refactor.

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 28 +++++++++++++++------
 source/dnode/vnode/src/tsdb/tsdbRead.c      | 10 +++++++-
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index a01a817087..e6210ff659 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -121,10 +121,10 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
 }
 
 // find the earliest block that contains the required records
-static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t backward) {
+static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t num, int32_t backward) {
   int32_t i = index;
   int32_t step = backward? 1:-1;
-  while (uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid && i >= 0) {
+  while (uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid && i >= 0 && i < num) {
     i += step;
   }
   return i - step;
@@ -146,7 +146,7 @@ static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64
 
   while (1) {
     if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) {
-      return findEarliestIndex(firstPos, uid, pBlockList, backward);
+      return findEarliestIndex(firstPos, uid, pBlockList, num, backward);
     }
 
     if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) {
@@ -161,15 +161,15 @@ static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64
     } else if (uid > pBlockList[midPos].maxUid) {
       firstPos = midPos + 1;
     } else {
-      return findEarliestIndex(midPos, uid, pBlockList, backward);
+      return findEarliestIndex(midPos, uid, pBlockList, num, backward);
     }
   }
 }
 
-static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t backward) {
+static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t num, int32_t backward) {
   int32_t i = index;
   int32_t step = backward? 1:-1;
-  while (uid == uidList[i] && i >= 0) {
+  while (uid == uidList[i] && i >= 0 && i < num) {
     i += step;
   }
   return i - step;
@@ -186,7 +186,7 @@ static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint
 
   while (1) {
     if (uid == uidList[firstPos]) {
-      return findEarliestRow(firstPos, uid, uidList, backward);
+      return findEarliestRow(firstPos, uid, uidList, num, backward);
     }
 
     if (uid > uidList[lastPos] || uid < uidList[firstPos]) {
@@ -201,7 +201,7 @@ static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint
     } else if (uid > uidList[midPos]) {
       firstPos = midPos + 1;
     } else {
-      return findEarliestRow(midPos, uid, uidList, backward);
+      return findEarliestRow(midPos, uid, uidList, num, backward);
     }
   }
 }
@@ -227,6 +227,18 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
       goto _exit;
+    } else {
+      size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+      SArray* pTmp = taosArrayInit(size, sizeof(SSttBlk));
+      for(int32_t i = 0; i < size; ++i) {
+        SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
+        if (p->suid == suid) {
+          taosArrayPush(pTmp, p);
+        }
+      }
+
+      taosArrayDestroy(pBlockLoadInfo->aSttBlk);
+      pBlockLoadInfo->aSttBlk = pTmp;
     }
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index d205ca13e2..d3053ca2b1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1429,6 +1429,9 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
       return TSDB_CODE_SUCCESS;
     } else {
       tRowMergerInit(&merge, &fRow, pReader->pSchema);
+
+      TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+      tRowMerge(&merge, &fRow1);
       doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge);
 
       // merge with block data if ts == key
@@ -3122,11 +3125,16 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
     SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i);
     SColData*        pData = tBlockDataGetColDataByIdx(pBlockData, j);
 
+    if (pData->cid < pCol->info.colId) {
+      j += 1;
+      continue;
+    }
+
     if (pData->cid == pCol->info.colId) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
-    } else {  // the specified column does not exist in file block, fill with null data
+    } else if (pData->cid > pCol->info.colId) {  // the specified column does not exist in file block, fill with null data
       colDataAppendNULL(pCol, outputRowIndex);
     }
 

From c5ec1741bc9d4e639b4d4207034e144cef31292c Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 7 Sep 2022 18:02:10 +0800
Subject: [PATCH 59/66] refactor(sync): add syncHeartbeatReply

---
 include/libs/sync/syncTools.h                 |  18 ++
 source/libs/sync/src/syncMessage.c            | 156 +++++++++++++++++-
 source/libs/sync/test/CMakeLists.txt          |  14 ++
 .../libs/sync/test/syncHeartbeatReplyTest.cpp | 105 ++++++++++++
 4 files changed, 292 insertions(+), 1 deletion(-)
 create mode 100644 source/libs/sync/test/syncHeartbeatReplyTest.cpp

diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h
index 09e24477d7..de2271554d 100644
--- a/include/libs/sync/syncTools.h
+++ b/include/libs/sync/syncTools.h
@@ -490,6 +490,24 @@ typedef struct SyncHeartbeatReply {
   int64_t  startTime;
 } SyncHeartbeatReply;
 
+SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId);
+void                syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg);
+void                syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen);
+void                syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg);
+char*               syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len);
+SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len);
+void                syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg);
+void                syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg);
+SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*              syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg);
+char*               syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg);
+
+// for debug ----------------------
+void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg);
+void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg);
+void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg);
+void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg);
+
 // ---------------------------------------------
 typedef struct SyncApplyMsg {
   uint32_t   bytes;
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index a16f87059f..faebe5bbec 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -1999,7 +1999,7 @@ SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) {
   memset(pMsg, 0, bytes);
   pMsg->bytes = bytes;
   pMsg->vgId = vgId;
-  pMsg->msgType = TDMT_SYNC_APPEND_ENTRIES;
+  pMsg->msgType = TDMT_SYNC_HEARTBEAT;
   return pMsg;
 }
 
@@ -2145,6 +2145,160 @@ void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) {
   }
 }
 
+// ---- message process SyncHeartbeatReply----
+SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId) {
+  uint32_t            bytes = sizeof(SyncHeartbeatReply);
+  SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes);
+  memset(pMsg, 0, bytes);
+  pMsg->bytes = bytes;
+  pMsg->vgId = vgId;
+  pMsg->msgType = TDMT_SYNC_HEARTBEAT_REPLY;
+  return pMsg;
+}
+
+void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg) {
+  if (pMsg != NULL) {
+    taosMemoryFree(pMsg);
+  }
+}
+
+void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen) {
+  ASSERT(pMsg->bytes <= bufLen);
+  memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg) {
+  memcpy(pMsg, buf, len);
+  ASSERT(len == pMsg->bytes);
+}
+
+char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len) {
+  char* buf = taosMemoryMalloc(pMsg->bytes);
+  ASSERT(buf != NULL);
+  syncHeartbeatReplySerialize(pMsg, buf, pMsg->bytes);
+  if (len != NULL) {
+    *len = pMsg->bytes;
+  }
+  return buf;
+}
+
+SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len) {
+  uint32_t            bytes = *((uint32_t*)buf);
+  SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes);
+  ASSERT(pMsg != NULL);
+  syncHeartbeatReplyDeserialize(buf, len, pMsg);
+  ASSERT(len == pMsg->bytes);
+  return pMsg;
+}
+
+void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg) {
+  memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+  pRpcMsg->msgType = pMsg->msgType;
+  pRpcMsg->contLen = pMsg->bytes;
+  pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+  syncHeartbeatReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg) {
+  syncHeartbeatReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+  SyncHeartbeatReply* pMsg = syncHeartbeatReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+  ASSERT(pMsg != NULL);
+  return pMsg;
+}
+
+cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg) {
+  char   u64buf[128] = {0};
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pMsg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+    cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+    cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+    cJSON* pSrcId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+    cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->srcId.addr;
+      cJSON*   pTmp = pSrcId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+    cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+    cJSON* pDestId = cJSON_CreateObject();
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+    cJSON_AddStringToObject(pDestId, "addr", u64buf);
+    {
+      uint64_t u64 = pMsg->destId.addr;
+      cJSON*   pTmp = pDestId;
+      char     host[128] = {0};
+      uint16_t port;
+      syncUtilU642Addr(u64, host, sizeof(host), &port);
+      cJSON_AddStringToObject(pTmp, "addr_host", host);
+      cJSON_AddNumberToObject(pTmp, "addr_port", port);
+    }
+    cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+    cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+    cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+    snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+    cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+    cJSON_AddStringToObject(pRoot, "matchIndex", u64buf);
+    snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime);
+    cJSON_AddStringToObject(pRoot, "startTime", u64buf);
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SyncHeartbeatReply", pRoot);
+  return pJson;
+}
+
+char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg) {
+  cJSON* pJson = syncHeartbeatReply2Json(pMsg);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg) {
+  char* serialized = syncHeartbeatReply2Str(pMsg);
+  printf("syncHeartbeatReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg) {
+  char* serialized = syncHeartbeatReply2Str(pMsg);
+  printf("syncHeartbeatReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg) {
+  char* serialized = syncHeartbeatReply2Str(pMsg);
+  sTrace("syncHeartbeatReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) {
+  if (gRaftDetailLog) {
+    char* serialized = syncHeartbeatReply2Str(pMsg);
+    sTrace("syncHeartbeatReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}
+
 // ---- message process SyncApplyMsg----
 SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) {
   uint32_t      bytes = sizeof(SyncApplyMsg) + dataLen;
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index e2be1523c0..b9cc7a391d 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -58,6 +58,7 @@ add_executable(syncReconfigFinishTest "")
 add_executable(syncRestoreFromSnapshot "")
 add_executable(syncRaftCfgIndexTest "")
 add_executable(syncHeartbeatTest "")
+add_executable(syncHeartbeatReplyTest "")
 
 
 target_sources(syncTest
@@ -300,6 +301,10 @@ target_sources(syncHeartbeatTest
     PRIVATE
     "syncHeartbeatTest.cpp"
 )
+target_sources(syncHeartbeatReplyTest
+    PRIVATE
+    "syncHeartbeatReplyTest.cpp"
+)
 
 
 target_include_directories(syncTest
@@ -602,6 +607,11 @@ target_include_directories(syncHeartbeatTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncHeartbeatReplyTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 
 
 target_link_libraries(syncTest
@@ -844,6 +854,10 @@ target_link_libraries(syncHeartbeatTest
     sync
     gtest_main
 )
+target_link_libraries(syncHeartbeatReplyTest
+    sync
+    gtest_main
+)
 
 
 enable_testing()
diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp
new file mode 100644
index 0000000000..0ccd7b70bb
--- /dev/null
+++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp
@@ -0,0 +1,105 @@
+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncHeartbeatReply *createMsg() {
+  SyncHeartbeatReply *pMsg = syncHeartbeatReplyBuild(1000);
+  pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+  pMsg->srcId.vgId = 100;
+  pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+  pMsg->destId.vgId = 100;
+
+  pMsg->term = 33;
+  pMsg->privateTerm = 44;
+  pMsg->startTime = taosGetTimestampMs();
+  return pMsg;
+}
+
+void test1() {
+  SyncHeartbeatReply *pMsg = createMsg();
+  syncHeartbeatReplyLog2((char *)"test1:", pMsg);
+  syncHeartbeatReplyDestroy(pMsg);
+}
+
+void test2() {
+  SyncHeartbeatReply *pMsg = createMsg();
+  uint32_t                len = pMsg->bytes;
+  char *                  serialized = (char *)taosMemoryMalloc(len);
+  syncHeartbeatReplySerialize(pMsg, serialized, len);
+  SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000);
+  syncHeartbeatReplyDeserialize(serialized, len, pMsg2);
+  syncHeartbeatReplyLog2((char *)"test2: syncHeartbeatReplySerialize -> syncHeartbeatReplyDeserialize ",
+                             pMsg2);
+
+  taosMemoryFree(serialized);
+  syncHeartbeatReplyDestroy(pMsg);
+  syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test3() {
+  SyncHeartbeatReply *pMsg = createMsg();
+  uint32_t                len;
+  char *                  serialized = syncHeartbeatReplySerialize2(pMsg, &len);
+  SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyDeserialize2(serialized, len);
+  syncHeartbeatReplyLog2((char *)"test3: syncHeartbeatReplySerialize3 -> syncHeartbeatReplyDeserialize2 ",
+                             pMsg2);
+
+  taosMemoryFree(serialized);
+  syncHeartbeatReplyDestroy(pMsg);
+  syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test4() {
+  SyncHeartbeatReply *pMsg = createMsg();
+  SRpcMsg                 rpcMsg;
+  syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg);
+  SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000);
+  syncHeartbeatReplyFromRpcMsg(&rpcMsg, pMsg2);
+  syncHeartbeatReplyLog2((char *)"test4: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg ",
+                             pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncHeartbeatReplyDestroy(pMsg);
+  syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test5() {
+  SyncHeartbeatReply *pMsg = createMsg();
+  SRpcMsg                 rpcMsg;
+  syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg);
+  SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyFromRpcMsg2(&rpcMsg);
+  syncHeartbeatReplyLog2((char *)"test5: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg2 ",
+                             pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncHeartbeatReplyDestroy(pMsg);
+  syncHeartbeatReplyDestroy(pMsg2);
+}
+
+int main() {
+  gRaftDetailLog = true;
+
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+
+  return 0;
+}

From 6c0598ec2f0fe4c45f4d7eaa50ff270b4011d049 Mon Sep 17 00:00:00 2001
From: Xuefeng Tan <1172915550@qq.com>
Date: Wed, 7 Sep 2022 18:14:34 +0800
Subject: [PATCH 60/66] fix(taosAdapter): wrong tmq offsetRest parameter
 (#16725)

---
 cmake/taosadapter_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index 6ff07fbb0d..eb0faf6d5d 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG 22bdac5
+        GIT_TAG 71e7ccf
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 582f44c2fdcfffc53251446bb8fd5f3be80b854e Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 7 Sep 2022 18:52:54 +0800
Subject: [PATCH 61/66] fix: fix statecount result error

---
 source/libs/function/src/builtinsimpl.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index f396f6e609..0d7fd1a6da 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -4711,7 +4711,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) {
       colDataAppendNULL(pOutput, i);
       // handle selectivity
       if (pCtx->subsidiaries.num > 0) {
-        appendSelectivityValue(pCtx, i, i);
+        appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
       }
       continue;
     }
@@ -4724,11 +4724,11 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) {
     } else {
       pInfo->count = 0;
     }
-    colDataAppend(pOutput, i, (char*)&output, false);
+    colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false);
 
     // handle selectivity
     if (pCtx->subsidiaries.num > 0) {
-      appendSelectivityValue(pCtx, i, i);
+      appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
     }
   }
 

From 3c952a337eb96ab68065a2a4ed54577d5233c8c6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 7 Sep 2022 18:54:58 +0800
Subject: [PATCH 62/66] fix(query): fix invalid read.

---
 source/dnode/vnode/src/tsdb/tsdbMergeTree.c |  4 ++--
 source/dnode/vnode/src/tsdb/tsdbRead.c      | 11 +++++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
index e6210ff659..d660dfa99d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
@@ -124,7 +124,7 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
 static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t num, int32_t backward) {
   int32_t i = index;
   int32_t step = backward? 1:-1;
-  while (uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid && i >= 0 && i < num) {
+  while (i >= 0 && i < num && uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid) {
     i += step;
   }
   return i - step;
@@ -169,7 +169,7 @@ static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64
 static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t num, int32_t backward) {
   int32_t i = index;
   int32_t step = backward? 1:-1;
-  while (uid == uidList[i] && i >= 0 && i < num) {
+  while (i >= 0 && i < num && uid == uidList[i]) {
     i += step;
   }
   return i - step;
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index d3053ca2b1..8229751d3c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3256,11 +3256,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl
     }
   }
 
+  // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here.
   if (pCond->suid != 0) {
-    pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->endVersion);
+    pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, /*pCond->endVersion*/ -1);
+    if (pReader->pSchema == NULL) {
+      tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr);
+    }
   } else if (taosArrayGetSize(pTableList) > 0) {
     STableKeyInfo* pKey = taosArrayGet(pTableList, 0);
-    pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->endVersion);
+    pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, /*pCond->endVersion*/ -1);
+    if (pReader->pSchema == NULL) {
+      tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr);
+    }
   }
 
   int32_t numOfTables = taosArrayGetSize(pTableList);

From 0a4644307ecb532bf7093c605b21f82a3a5e6de5 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 7 Sep 2022 19:48:43 +0800
Subject: [PATCH 63/66] fix: fix compile error

---
 source/libs/transport/src/transCli.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index dc2d937c49..62cdf4e784 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -151,8 +151,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
 static FORCE_INLINE void destroyUserdata(STransMsg* userdata);
 static FORCE_INLINE void destroyCmsg(void* cmsg);
 static FORCE_INLINE int  cliRBChoseIdx(STrans* pTransInst);
+static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
 
-static void transDestroyConnCtx(STransConnCtx* ctx);
 // thread obj
 static SCliThrd* createThrdObj();
 static void      destroyThrdObj(SCliThrd* pThrd);
@@ -1124,7 +1124,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
   return cli;
 }
 
-FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
+static FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
   if (userdata->pCont == NULL) {
     return;
   }
@@ -1132,7 +1132,7 @@ FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
   userdata->pCont = NULL;
 }
 
-FORCE_INLINE void destroyCmsg(void* arg) {
+static FORCE_INLINE void destroyCmsg(void* arg) {
   SCliMsg* pMsg = arg;
   if (pMsg == NULL) {
     return;
@@ -1198,7 +1198,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
   taosMemoryFree(pThrd);
 }
 
-static void transDestroyConnCtx(STransConnCtx* ctx) {
+static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx) {
   //
   taosMemoryFree(ctx);
 }

From 6bb23a8f566eaab85665b935ceda6b0c378a7de5 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Wed, 7 Sep 2022 23:31:40 +0800
Subject: [PATCH 64/66] fix(sync): adjust message define

---
 include/common/tmsgdef.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 33d01bc46d..3f917ff0d1 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -261,8 +261,6 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES, "sync-append-entries", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES_BATCH, "sync-append-entries-batch", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES_REPLY, "sync-append-entries-reply", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_NOOP, "sync-noop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_UNKNOWN, "sync-unknown", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_COMMON_RESPONSE, "sync-common-response", NULL, NULL)
@@ -274,6 +272,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   
 #if defined(TD_MSG_NUMBER_)

From 684b390c46b2d77aba0c761979d776c568071a23 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Thu, 8 Sep 2022 00:33:48 +0800
Subject: [PATCH 65/66] fix: build tdengine on apple silicon

---
 cmake/cmake.platform | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/cmake.platform b/cmake/cmake.platform
index 3aa1ffc07e..e4d440d76e 100644
--- a/cmake/cmake.platform
+++ b/cmake/cmake.platform
@@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "")
     SET(TD_ARM_32 TRUE)
     ADD_DEFINITIONS("-D_TD_ARM_")
     ADD_DEFINITIONS("-D_TD_ARM_32")
-  ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+  ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
     MESSAGE(STATUS "The current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     SET(TD_ARM_64 TRUE)

From 714dcbaac88625692d0b61783a45108d4c4fc715 Mon Sep 17 00:00:00 2001
From: "GL.Ruan" <ruanguiliang_34@whu.edu.cn>
Date: Thu, 8 Sep 2022 09:59:13 +0800
Subject: [PATCH 66/66] delete redundant period (#16738)

---
 docs/zh/05-get-started/01-docker.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md
index e2be419517..35e7ad1933 100644
--- a/docs/zh/05-get-started/01-docker.md
+++ b/docs/zh/05-get-started/01-docker.md
@@ -62,7 +62,7 @@ taos>
 
 ## 体验查询
 
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。
+使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。
 
 查询超级表下记录总条数：