Merge branch 'main' into lihui3.0/test

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 0681d8b
+        GIT_TAG 4378702
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

include/common/tcommon.h

@@ -341,6 +341,8 @@ typedef struct {
     float       f;
   };
   size_t length;
+  bool keyEscaped;
+  bool valueEscaped;
 } SSmlKv;
 
 #define QUERY_ASC_FORWARD_STEP  1

include/common/tmsg.h

@@ -689,6 +689,7 @@ typedef struct {
 
 int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
 int32_t tDeserializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
+void    tFreeSAlterUserReq(SAlterUserReq* pReq);
 
 typedef struct {
   char user[TSDB_USER_LEN];

include/common/ttokendef.h

@@ -346,6 +346,7 @@
 #define TK_VIEW                           328
 #define TK_WAL                            329
 
+
 #define TK_NK_SPACE   600
 #define TK_NK_COMMENT 601
 #define TK_NK_ILLEGAL 602

include/libs/catalog/catalog.h

@@ -82,6 +82,7 @@ typedef struct SCatalogReq {
   SArray* pUser;          // element is SUserAuthInfo
   SArray* pTableIndex;    // element is SNAME
   SArray* pTableCfg;      // element is SNAME
+  SArray* pTableTag;      // element is SNAME
   bool    qNodeRequired;  // valid qnode
   bool    dNodeRequired;  // valid dnode
   bool    svrVerRequired;
@@ -105,6 +106,7 @@ typedef struct SMetaData {
   SArray*   pUser;        // pRes = SUserAuthRes*
   SArray*   pQnodeList;   // pRes = SArray<SQueryNodeLoad>*
   SArray*   pTableCfg;    // pRes = STableCfg*
+  SArray*   pTableTag;    // pRes = SArray<STagVal>*
   SArray*   pDnodeList;   // pRes = SArray<SEpSet>*
   SMetaRes* pSvrVer;      // pRes = char*
 } SMetaData;
@@ -122,8 +124,8 @@ typedef struct SSTableVersion {
   char     stbName[TSDB_TABLE_NAME_LEN];
   uint64_t dbId;
   uint64_t suid;
-  int16_t  sversion;
+  int32_t  sversion;
-  int16_t  tversion;
+  int32_t  tversion;
   int32_t  smaVer;
 } SSTableVersion;
 
@@ -312,6 +314,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char*
 
 int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
 
+int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
+
 int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg);
 
 int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp);

include/libs/nodes/querynodes.h

@@ -379,6 +379,8 @@ typedef struct SVnodeModifyOpStmt {
   SName                 usingTableName;
   const char*           pBoundCols;
   struct STableMeta*    pTableMeta;
+  SNode*                pTagCond;
+  SArray*               pTableTag;
   SHashObj*             pVgroupsHashObj;
   SHashObj*             pTableBlockHashObj;  // SHashObj<tuid, STableDataCxt*>
   SHashObj*             pSubTableHashObj;

include/libs/qcom/query.h

@@ -116,8 +116,8 @@ typedef struct STableMeta {
 
   // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
   // info
-  int16_t       sversion;
+  int32_t       sversion;
-  int16_t       tversion;
+  int32_t       tversion;
   STableComInfo tableInfo;
   SSchema       schema[];
 } STableMeta;

include/libs/stream/tstream.h

@@ -273,6 +273,7 @@ typedef struct SStreamId {
 typedef struct SCheckpointInfo {
   int64_t id;
   int64_t version;   // offset in WAL
+  int64_t currentVer;// current offset in WAL, not serialize it
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
@@ -537,6 +538,7 @@ void    streamTaskInputFail(SStreamTask* pTask);
 int32_t streamTryExec(SStreamTask* pTask);
 int32_t streamSchedExec(SStreamTask* pTask);
 int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
+bool    streamTaskShouldStop(const SStreamStatus* pStatus);
 
 int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);
 

include/util/tdef.h

@@ -368,11 +368,11 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_STT_TRIGGER            1
 #define TSDB_MAX_STT_TRIGGER            16
 #define TSDB_DEFAULT_SST_TRIGGER        1
-#define TSDB_MIN_HASH_PREFIX            0
+#define TSDB_MIN_HASH_PREFIX            (2 - TSDB_TABLE_NAME_LEN)
-#define TSDB_MAX_HASH_PREFIX            128
+#define TSDB_MAX_HASH_PREFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_PREFIX        0
-#define TSDB_MIN_HASH_SUFFIX            0
+#define TSDB_MIN_HASH_SUFFIX            (2 - TSDB_TABLE_NAME_LEN)
-#define TSDB_MAX_HASH_SUFFIX            128
+#define TSDB_MAX_HASH_SUFFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_SUFFIX        0
 
 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1

include/util/tutil.h

@@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
 
 static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
                                              int32_t suffix) {
-  if (prefix == 0 && suffix == 0) {
+  if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) {
     return MurmurHash3_32(tbname, tblen);
-  } else {
+  } else if (prefix > 0 || suffix > 0) {
-    if (tblen <= (prefix + suffix)) {
-      return MurmurHash3_32(tbname, tblen);
-    } else {
     return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
+  } else {
+    char tbName[TSDB_TABLE_FNAME_LEN];
+    int32_t offset = 0;
+    if (prefix < 0) {
+      offset = -1 * prefix;
+      strncpy(tbName, tbname, offset);
     }
+    if (suffix < 0) {
+      strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix);
+      offset += -1 *suffix;
+    }
+    return MurmurHash3_32(tbName, offset);
   }
 }
 
@@ -98,6 +106,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
     goto LABEL;                            \
   }
 
+#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)
+
 #ifdef __cplusplus
 }
 #endif

packaging/deb/DEBIAN/preinst

@@ -26,6 +26,38 @@ if pidof taosd &> /dev/null; then
     sleep 1
 fi
 
+# Stop adapter service if running
+if pidof taosadapter &> /dev/null; then
+    if pidof systemd &> /dev/null; then
+        ${csudo}systemctl stop taosadapter || :
+    elif $(which service  &> /dev/null); then
+        ${csudo}service taosadapter stop || :
+    else
+        pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}')
+        if [ -n "$pid" ]; then
+           ${csudo}kill -9 $pid   || :
+        fi
+    fi
+    echo "Stop taosadapter service success!"
+    sleep 1
+fi
+
+# Stop keeper service if running
+if pidof taoskeeper &> /dev/null; then
+    if pidof systemd &> /dev/null; then
+        ${csudo}systemctl stop taoskeeper || :
+    elif $(which service  &> /dev/null); then
+        ${csudo}service taoskeeper stop || :
+    else
+        pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
+        if [ -n "$pid" ]; then
+           ${csudo}kill -9 $pid   || :
+        fi
+    fi
+    echo "Stop taoskeeper service success!"
+    sleep 1
+fi
+
 # if taos.cfg already softlink, remove it
 cfg_install_dir="/etc/taos"
 install_main_dir="/usr/local/taos"
@@ -41,6 +73,11 @@ if [ -f "${install_main_dir}/taosadapter.service" ]; then
     ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || :
 fi
 
+if [ -f "${install_main_dir}/taoskeeper.toml" ]; then
+    ${csudo}rm -f ${install_main_dir}/cfg/taoskeeper.toml     || :
+fi
+
+
 # there can not libtaos.so*, otherwise ln -s  error
 ${csudo}rm -f ${install_main_dir}/driver/libtaos.*   || :
 [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so   || :

packaging/deb/DEBIAN/prerm

@@ -32,6 +32,7 @@ else
     ${csudo}rm -f ${bin_link_dir}/udfd       || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
     ${csudo}rm -f ${bin_link_dir}/taosdemo   || :
+    ${csudo}rm -f ${bin_link_dir}/taoskeeper   || :
     ${csudo}rm -f ${cfg_link_dir}/*          || :
     ${csudo}rm -f ${inc_link_dir}/taos.h     || :
     ${csudo}rm -f ${inc_link_dir}/taosdef.h     || :

packaging/deb/makedeb.sh

@@ -44,8 +44,31 @@ mkdir -p ${pkg_dir}${install_home_path}/include
 #mkdir -p ${pkg_dir}${install_home_path}/init.d
 mkdir -p ${pkg_dir}${install_home_path}/script
 
+# download taoskeeper and build
+if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then
+  arch=amd64
+elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then
+  arch=386
+elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then
+  arch=arm
+elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then
+  arch=arm64
+else
+  arch=$cpuType
+fi
+
+echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
+echo "$top_dir=${top_dir}"
+taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
+echo "taoskeeper_binary: ${taoskeeper_binary}"
+
+# copy config files
+cp $(dirname ${taoskeeper_binary})/config/taoskeeper.toml ${pkg_dir}${install_home_path}/cfg
+cp $(dirname ${taoskeeper_binary})/taoskeeper.service ${pkg_dir}${install_home_path}/cfg
+
 cp ${compile_dir}/../packaging/cfg/taos.cfg         ${pkg_dir}${install_home_path}/cfg
 cp ${compile_dir}/../packaging/cfg/taosd.service    ${pkg_dir}${install_home_path}/cfg
+
 if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
     cp ${compile_dir}/test/cfg/taosadapter.toml		${pkg_dir}${install_home_path}/cfg || :
 fi
@@ -53,6 +76,7 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
     cp ${compile_dir}/test/cfg/taosadapter.service	${pkg_dir}${install_home_path}/cfg || :
 fi
 
+cp ${taoskeeper_binary}                      ${pkg_dir}${install_home_path}/bin
 #cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_path}/init.d
 cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
@@ -143,6 +167,7 @@ else
   exit 1
 fi
 
+rm -rf ${pkg_dir}/build-taoskeeper
 # make deb package
 dpkg -b ${pkg_dir} $debname
 echo "make deb package success!"
@@ -150,4 +175,5 @@ echo "make deb package success!"
 cp ${pkg_dir}/*.deb ${output_dir}
 
 # clean temp dir
+
 rm -rf ${pkg_dir}

packaging/rpm/makerpm.sh

@@ -35,14 +35,16 @@ function cp_rpm_package() {
     local cur_dir
     cd $1
     cur_dir=$(pwd)
-
+    echo "cp_rpm_package cd: ${cur_dir}"
     for dirlist in "$(ls ${cur_dir})"; do
         if test -d ${dirlist}; then
             cd ${dirlist}
+            echo 'cp_rpm_package ${cur_dir}/${dirlist}'
             cp_rpm_package ${cur_dir}/${dirlist}
             cd ..
         fi
         if test -e ${dirlist}; then
+            echo "${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm"
             cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm
         fi
     done
@@ -54,6 +56,25 @@ fi
 ${csudo}mkdir -p ${pkg_dir}
 cd ${pkg_dir}
 
+# download taoskeeper and build
+if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then
+  arch=amd64
+elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then
+  arch=386
+elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then
+  arch=arm
+elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then
+  arch=arm64
+else
+  arch=$cpuType
+fi
+
+cd ${top_dir}
+echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
+taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
+echo "taoskeeper_binary: ${taoskeeper_binary}"
+cd ${package_dir}
+
 ${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS
 
 ${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file}
@@ -85,3 +106,4 @@ mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname}
 
 cd ..
 ${csudo}rm -rf ${pkg_dir}
+rm -rf ${top_dir}/build-taoskeeper

packaging/rpm/tdengine.spec

@@ -3,6 +3,7 @@
 %define cfg_install_dir  /etc/taos
 %define __strip /bin/true
 %global __python /usr/bin/python3
+%global _build_id_links none
 
 Name:		tdengine
 Version:	%{_version}
@@ -62,6 +63,15 @@ fi
 if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then
     cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg
 fi
+
+if [ -f %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml ]; then
+    cp %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml %{buildroot}%{homepath}/cfg ||:
+fi
+
+if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper.service ]; then
+    cp %{_compiledir}/../build-taoskeeper/taoskeeper.service %{buildroot}%{homepath}/cfg ||:
+fi
+
 #cp %{_compiledir}/../packaging/rpm/taosd            %{buildroot}%{homepath}/init.d
 cp %{_compiledir}/../packaging/tools/post.sh        %{buildroot}%{homepath}/script
 cp %{_compiledir}/../packaging/tools/preun.sh       %{buildroot}%{homepath}/script
@@ -73,8 +83,12 @@ cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/udfd                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosBenchmark           %{buildroot}%{homepath}/bin
 
+if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper ]; then
+    cp %{_compiledir}/../build-taoskeeper/taoskeeper %{buildroot}%{homepath}/bin
+fi
+
 if [ -f %{_compiledir}/build/bin/taosadapter ]; then
-    cp %{_compiledir}/build/bin/taosadapter                    %{buildroot}%{homepath}/bin ||:
+    cp %{_compiledir}/build/bin/taosadapter                    %{buildroot}%{homepath}/bin
 fi
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile}            %{buildroot}%{homepath}/driver ||:
@@ -119,7 +133,9 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then
         cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig
     fi
 fi
-
+ls -al %{buildroot}%{homepath}/bin
+tree -L 5
+echo "==============================copying files done"
 #Scripts executed before installation
 %pre
 if [ -f /var/lib/taos/dnode/dnodeCfg.json ]; then
@@ -196,6 +212,7 @@ if [ $1 -eq 0 ];then
     ${csudo}rm -f ${bin_link_dir}/taosd      || :
     ${csudo}rm -f ${bin_link_dir}/udfd       || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
+    ${csudo}rm -f ${bin_link_dir}/taoskeeper       || :
     ${csudo}rm -f ${cfg_link_dir}/*          || :
     ${csudo}rm -f ${inc_link_dir}/taos.h     || :
     ${csudo}rm -f ${inc_link_dir}/taosdef.h     || :

packaging/tools/makeclient.sh

@@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then
       mkdir -p ${install_dir}/connector
       if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
           if [ "$osType" != "Darwin" ]; then
-              [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
+              jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
+              [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
           fi
           git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
           rm -rf ${install_dir}/connector/go/.git ||:

packaging/tools/makepkg.sh

@@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
     connector_dir="${code_dir}/connector"
     mkdir -p ${install_dir}/connector
     if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
-        [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
+        tmp_pwd=`pwd`
+    	  cd ${install_dir}/connector
+    	  if [ ! -d taos-connector-jdbc ];then
+          	git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||:
+    	  fi
+    	  cd taos-connector-jdbc
+    	  mvn clean package -Dmaven.test.skip=true
+    	  echo  ${build_dir}/lib/
+    	  cp target/*.jar  ${build_dir}/lib/
+    	  cd ${install_dir}/connector
+    	  rm -rf taos-connector-jdbc
+    	  cd ${tmp_pwd}
+   	    jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
+        [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
         git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
         rm -rf ${install_dir}/connector/go/.git ||:
 

packaging/tools/post.sh

@@ -436,7 +436,7 @@ function local_fqdn_check() {
 
 function install_taosadapter_config() {
     if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then
-        [ ! -d %{cfg_install_dir} ] &&
+        [ ! -d ${cfg_install_dir} ] &&
             ${csudo}${csudo}mkdir -p ${cfg_install_dir}
         [ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir}
         [ -f ${cfg_install_dir}/taosadapter.toml ] &&
@@ -451,19 +451,26 @@ function install_taosadapter_config() {
 }
 
 function install_taoskeeper_config() {
-    if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then
+    # if new environment without taoskeeper
-        [ ! -d %{cfg_install_dir} ] &&
+    if [[ ! -f "${cfg_install_dir}/keeper.toml" ]] && [[ ! -f "${cfg_install_dir}/taoskeeper.toml" ]]; then
-            ${csudo}${csudo}mkdir -p ${cfg_install_dir}
+        [ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir}
-        [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir}
+        [ -f ${cfg_dir}/taoskeeper.toml ] && ${csudo}cp ${cfg_dir}/taoskeeper.toml ${cfg_install_dir}
-        [ -f ${cfg_install_dir}/keeper.toml ] &&
+        [ -f ${cfg_install_dir}/taoskeeper.toml ] &&
-            ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml
+            ${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml
+    fi
+    # if old machine with taoskeeper.toml file
+    if [ -f ${cfg_install_dir}/taoskeeper.toml ]; then
+        ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new
     fi
 
-    [ -f ${cfg_dir}/keeper.toml ] &&
+    if [ -f ${cfg_install_dir}/keeper.toml ]; then
-        ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new
+        echo "The file keeper.toml will be renamed to taoskeeper.toml"
+        ${csudo}mv ${cfg_install_dir}/keeper.toml ${cfg_install_dir}/taoskeeper.toml
+        ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new
+    fi
 
-    [ -f ${cfg_install_dir}/keeper.toml ] &&
+    [ -f ${cfg_install_dir}/taoskeeper.toml ] &&
-        ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir}
+        ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir}
 }
 
 function install_config() {
@@ -655,6 +662,15 @@ function install_taosadapter_service() {
     fi
 }
 
+function install_taoskeeper_service() {
+    if ((${service_mod}==0)); then
+        [ -f ${script_dir}/../cfg/taoskeeper.service ] &&\
+            ${csudo}cp ${script_dir}/../cfg/taoskeeper.service \
+            ${service_config_dir}/ || :
+        ${csudo}systemctl daemon-reload
+    fi
+}
+
 function install_service() {
   log_print "start install service"
   if [ "$osType" != "Darwin" ]; then
@@ -732,6 +748,7 @@ function install_TDengine() {
     install_taosadapter_config
     install_taoskeeper_config
     install_taosadapter_service
+    install_taoskeeper_service
     install_service
     install_app
 

packaging/tools/preun.sh

@@ -17,7 +17,7 @@ cfg_link_dir="/usr/local/taos/cfg"
 
 service_config_dir="/etc/systemd/system"
 taos_service_name="taosd"
-
+taoskeeper_service_name="taoskeeper"
 csudo=""
 if command -v sudo > /dev/null; then
     csudo="sudo "
@@ -57,6 +57,13 @@ function kill_taosd() {
   fi
 }
 
+function kill_taoskeeper() {
+  pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo}kill -9 $pid   || :
+  fi
+}
+
 function clean_service_on_systemd() {
     taosadapter_service_config="${service_config_dir}/taosadapter.service"
     if systemctl is-active --quiet taosadapter; then
@@ -76,6 +83,12 @@ function clean_service_on_systemd() {
 
     [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
 
+    taoskeeper_service_config="${service_config_dir}/${taoskeeper_service_name}.service"
+    if systemctl is-active --quiet ${taoskeeper_service_name}; then
+        echo "TDengine taoskeeper is running, stopping it..."
+        ${csudo}systemctl stop ${taoskeeper_service_name} &> /dev/null || echo &> /dev/null
+    fi
+    [ -f ${taoskeeper_service_config} ] && ${csudo}rm -f ${taoskeeper_service_config}
 }
 
 function clean_service_on_sysvinit() {
@@ -111,6 +124,7 @@ function clean_service() {
         # must manual stop taosd
         kill_taosadapter
         kill_taosd
+        kill_taoskeeper
     fi
 }
 
@@ -124,6 +138,7 @@ ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
 ${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
 ${csudo}rm -f ${bin_link_dir}/taosdemo   || :
 ${csudo}rm -f ${bin_link_dir}/set_core   || :
+${csudo}rm -f ${bin_link_dir}/taoskeeper  || :
 ${csudo}rm -f ${cfg_link_dir}/*.new      || :
 ${csudo}rm -f ${inc_link_dir}/taos.h     || :
 ${csudo}rm -f ${inc_link_dir}/taosdef.h  || :

source/client/inc/clientSml.h

@@ -70,7 +70,7 @@ extern "C" {
 #define VALUE_LEN 6
 
 #define OTD_JSON_FIELDS_NUM     4
-#define MAX_RETRY_TIMES 100
+#define MAX_RETRY_TIMES 10
 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
 
 typedef enum {
@@ -107,6 +107,7 @@ typedef struct {
   int32_t colsLen;
   int32_t timestampLen;
 
+  bool    measureEscaped;
   SArray *colArray;
 } SSmlLineInfo;
 
@@ -206,6 +207,19 @@ typedef struct {
 
 #define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
 
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
+
 extern int64_t smlFactorNS[3];
 extern int64_t smlFactorS[3];
 
@@ -237,6 +251,7 @@ uint8_t           smlGetTimestampLen(int64_t num);
 void              clearColValArray(SArray* pCols);
 void              smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag);
 
+void freeSSmlKv(void* data);
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);

source/client/src/clientMain.c

@@ -741,6 +741,7 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
   taosArrayDestroy(pCatalogReq->pUser);
   taosArrayDestroy(pCatalogReq->pTableIndex);
   taosArrayDestroy(pCatalogReq->pTableCfg);
+  taosArrayDestroy(pCatalogReq->pTableTag);
   taosMemoryFree(pCatalogReq);
 }
 
@@ -975,8 +976,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 
   if (TSDB_CODE_SUCCESS == code) {
     pRequest->stmtType = pRequest->pQuery->pRoot->type;
-    phaseAsyncQuery(pWrapper);
+    code = phaseAsyncQuery(pWrapper);
-  } else {
+  }
+
+  if (TSDB_CODE_SUCCESS != code) {
     tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
              pRequest->requestId);
     destorySqlCallbackWrapper(pWrapper);
@@ -1273,7 +1276,6 @@ _return:
   return code;
 }
 
-
 int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;

source/client/src/clientSml.c

@@ -535,7 +535,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
   if (index) {
     if (colField[*index].type != kv->type) {
       uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      return TSDB_CODE_SML_INVALID_DATA;
     }
 
     if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR &&
@@ -765,8 +765,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
 
     size_t superTableLen = 0;
     void  *superTable = taosHashGetKey(tmp, &superTableLen);
+    char* measure = taosMemoryMalloc(superTableLen);
+    memcpy(measure, superTable, superTableLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen);
     memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
-    memcpy(pName.tname, superTable, superTableLen);
+    memcpy(pName.tname, measure, superTableLen);
+    taosMemoryFree(measure);
 
     code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
 
@@ -1049,7 +1053,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
   //  }
   //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
-  taosArrayDestroy(tag->tags);
+  taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
 }
 
@@ -1063,6 +1067,12 @@ void clearColValArray(SArray *pCols) {
   }
 }
 
+void freeSSmlKv(void* data){
+  SSmlKv *kv = (SSmlKv*)data;
+  if(kv->keyEscaped) taosMemoryFree((void*)(kv->key));
+  if(kv->valueEscaped) taosMemoryFree((void*)(kv->value));
+}
+
 void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
@@ -1098,11 +1108,11 @@ void smlDestroyInfo(SSmlHandle *info) {
   }
   taosArrayDestroy(info->valueJsonArray);
 
-  taosArrayDestroy(info->preLineTagKV);
+  taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv);
 
   if (!info->dataFormat) {
     for (int i = 0; i < info->lineNum; i++) {
-      taosArrayDestroy(info->lines[i].colArray);
+      taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
@@ -1165,8 +1175,10 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
   }
   for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
+    terrno = 0;
     taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
     if (terrno == TSDB_CODE_DUP_KEY) {
+      taosHashCleanup(kvHash);
       return terrno;
     }
   }
@@ -1240,12 +1252,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
+      taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
+      terrno = 0;
       smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
       if (terrno == TSDB_CODE_DUP_KEY) {
         return terrno;
       }
       smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
-      taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
     }
   }
   uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
@@ -1304,9 +1317,15 @@ static int32_t smlInsertData(SSmlHandle *info) {
     uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
            tableData->uid, info->dataFormat);
 
+    int   measureLen = tableData->sTableNameLen;
+    char* measure = (char*)taosMemoryMalloc(tableData->sTableNameLen);
+    memcpy(measure, tableData->sTableName, tableData->sTableNameLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+
     code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
-                       (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
+                       (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen,
                        info->ttl, info->msgBuf.buf, info->msgBuf.len);
+    taosMemoryFree(measure);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
       return code;
@@ -1420,14 +1439,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
 
     char cTmp = 0;  // for print tmp if is raw
     if (info->isRawLine) {
-      cTmp = tmp[len - 1];
+      cTmp = tmp[len];
-      tmp[len - 1] = '\0';
+      tmp[len] = '\0';
     }
 
     uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
            info->isRawLine, numLines, info->protocol, len, tmp);
     if (info->isRawLine) {
-      tmp[len - 1] = cTmp;
+      tmp[len] = cTmp;
     }
 
     if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
@@ -1449,6 +1468,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
       code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
     }
     if (code != TSDB_CODE_SUCCESS) {
+      tmp[len] = '\0';
       uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
       return code;
     }
@@ -1494,8 +1514,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
 
   do {
     code = smlModifyDBSchemas(info);
-    if (code == 0) break;
+    if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break;
-    taosMsleep(500);
+    taosMsleep(100);
     uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
   } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);
 

source/client/src/clientSmlLine.c

@@ -21,32 +21,33 @@
 #include "clientSml.h"
 
 // comma ,
-// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH)
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
 // space
-// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH)
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
 // equal =
-// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH)
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
 // quote "
-// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH)
+//#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
-#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
 // SLASH
-// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH)
 
-#define IS_SLASH_LETTER(sql)                                                                           \
+#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \
+  (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
-                           *(sql) == SLASH))  //  (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) ||
-                                              //  IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql))
 
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql)                                                         \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL))
 
-#define PROCESS_SLASH(key, keyLen)           \
+#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen)           \
   for (int i = 1; i < keyLen; ++i) {                        \
-    if (IS_SLASH_LETTER(key + i)) {          \
+    if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
+
+#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) {          \
       MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      i--;                                   \
       keyLen--;                              \
     }                                        \
   }
@@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
       SSmlSTableMeta *sMeta = NULL;
       if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
+        char* measure = currElement->measure;
+        int   measureLen = currElement->measureLen;
+        if(currElement->measureEscaped){
+          measure = (char*)taosMemoryMalloc(currElement->measureLen);
+          memcpy(measure, currElement->measure, currElement->measureLen);
+          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+        }
+        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+        if(currElement->measureEscaped){
+          taosMemoryFree(measure);
+        }
         if (pTableMeta == NULL) {
           info->dataFormat = false;
           info->reRun = true;
@@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       info->maxTagKVs = (*tmp)->tags;
     }
   }
-  taosArrayClear(preLineKV);
+  taosArrayClearEx(preLineKV, freeSSmlKv);
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
     }
 
-    bool hasSlash = false;
     // parse key
     const char *key = *sql;
     size_t      keyLen = 0;
+    bool        keyEscaped = false;
+    size_t      keyLenEscaped = 0;
     while (*sql < sqlEnd) {
       if (unlikely(IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         (*sql)++;
         break;
       }
-      if (!hasSlash) {
+      if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
-        hasSlash = (*(*sql) == SLASH);
+        keyLenEscaped++;
+        keyEscaped = true;
       }
       (*sql)++;
     }
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(key, keyLen)
-    }
 
-    if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
+    if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
@@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     // parse value
     const char *value = *sql;
     size_t      valueLen = 0;
-    hasSlash = false;
+    bool        valueEscaped = false;
+    size_t      valueLenEscaped = 0;
     while (*sql < sqlEnd) {
       // parse value
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
@@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         return TSDB_CODE_SML_INVALID_DATA;
       }
 
-      if (!hasSlash) {
+      if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
-        hasSlash = (*(*sql) == SLASH);
+        valueLenEscaped++;
+        valueEscaped = true;
       }
 
       (*sql)++;
@@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       return TSDB_CODE_SML_INVALID_DATA;
     }
 
-    if (unlikely(hasSlash)) {
+    if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      PROCESS_SLASH(value, valueLen)
-    }
-
-    if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
       return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
+    if (keyEscaped){
+      char *tmp = (char*)taosMemoryMalloc(keyLen);
+      memcpy(tmp, key, keyLen);
+      PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
+      key = tmp;
+    }
+    if (valueEscaped){
+      char *tmp = (char*)taosMemoryMalloc(valueLen);
+      memcpy(tmp, value, valueLen);
+      PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
+      value = tmp;
+    }
+    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped};
+    taosArrayPush(preLineKV, &kv);
     if (info->dataFormat) {
       if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
         info->dataFormat = false;
@@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         info->needModifySchema = true;
       }
     }
-    taosArrayPush(preLineKV, &kv);
 
     cnt++;
     if (IS_SPACE(*sql)) {
@@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   tinfo->tags = taosArrayDup(preLineKV, NULL);
+  for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){
+    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
+    if(kv->keyEscaped)kv->key = NULL;
+    if(kv->valueEscaped)kv->value = NULL;
+  }
 
   smlSetCTableName(tinfo);
   tinfo->uid = info->uid++;
@@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       SSmlSTableMeta **tmp =
           (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
       if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
+        char* measure = currElement->measure;
+        int   measureLen = currElement->measureLen;
+        if(currElement->measureEscaped){
+          measure = (char*)taosMemoryMalloc(currElement->measureLen);
+          memcpy(measure, currElement->measure, currElement->measureLen);
+          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+        }
+        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+        if(currElement->measureEscaped){
+          taosMemoryFree(measure);
+        }
         if (pTableMeta == NULL) {
           info->dataFormat = false;
           info->reRun = true;
@@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       break;
     }
 
-    bool hasSlash = false;
     // parse key
     const char *key = *sql;
     size_t      keyLen = 0;
+    bool        keyEscaped = false;
+    size_t      keyLenEscaped = 0;
     while (*sql < sqlEnd) {
       if (unlikely(IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         (*sql)++;
         break;
       }
-      if (!hasSlash) {
+      if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
-        hasSlash = (*(*sql) == SLASH);
+        keyLenEscaped++;
+        keyEscaped = true;
       }
       (*sql)++;
     }
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(key, keyLen)
-    }
 
-    if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
+    if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
@@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     // parse value
     const char *value = *sql;
     size_t      valueLen = 0;
-    hasSlash = false;
+    bool        valueEscaped = false;
+    size_t      valueLenEscaped = 0;
     bool        isInQuote = false;
+    const char *escapeChar = NULL;
     while (*sql < sqlEnd) {
       // parse value
-      if (unlikely(IS_QUOTE(*sql))) {
+      if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) {
         isInQuote = !isInQuote;
         (*sql)++;
         continue;
@@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       if (!isInQuote) {
         if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
           break;
-        } else if (unlikely(IS_EQUAL(*sql))) {
-          smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-          return TSDB_CODE_SML_INVALID_DATA;
         }
       }
-      if (!hasSlash) {
+      if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) {
-        hasSlash = (*(*sql) == SLASH);
+        escapeChar = *sql;
+        valueEscaped = true;
+        valueLenEscaped++;
       }
 
       (*sql)++;
@@ -416,9 +451,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
       return TSDB_CODE_SML_INVALID_DATA;
     }
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(value, valueLen)
-    }
 
     SSmlKv  kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen};
     int32_t ret = smlParseValue(&kv, &info->msgBuf);
@@ -427,11 +459,28 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       return ret;
     }
 
+    if (keyEscaped){
+      char *tmp = (char*)taosMemoryMalloc(kv.keyLen);
+      memcpy(tmp, key, kv.keyLen);
+      PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
+      kv.key = tmp;
+      kv.keyEscaped = keyEscaped;
+    }
+
+    if (valueEscaped){
+      char *tmp = (char*)taosMemoryMalloc(kv.length);
+      memcpy(tmp, kv.value, kv.length);
+      PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
+      kv.value = tmp;
+      kv.valueEscaped = valueEscaped;
+    }
+
     if (info->dataFormat) {
       // cnt begin 0, add ts so + 2
       if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       // bind data
@@ -440,22 +489,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         uDebug("smlBuildCol error, retry");
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       if (cnt >= taosArrayGetSize(info->masColKVs)) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt);
       if (kv.type != maxKV->type) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       if (unlikely(!IS_SAME_KEY)) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
 
@@ -463,6 +516,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         maxKV->length = kv.length;
         info->needModifySchema = true;
       }
+      freeSSmlKv(&kv);
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -487,10 +541,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   elements->measure = sql;
 
   // parse measure
+  size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
-    if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) {
+    if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
-      MOVE_FORWARD_ONE(sql, sqlEnd - sql);
+      elements->measureEscaped = true;
-      sqlEnd--;
+      measureLenEscaped++;
+      sql++;
       continue;
     }
     if (unlikely(IS_COMMA(sql))) {
@@ -503,7 +559,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     sql++;
   }
   elements->measureLen = sql - elements->measure;
-  if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) {
+  if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) {
     smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL);
     return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
   }
@@ -581,7 +637,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
                .keyLen = TS_LEN,
                .type = TSDB_DATA_TYPE_TIMESTAMP,
                .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
+               .keyEscaped = false,
+               .valueEscaped = false};
   if (info->dataFormat) {
     uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
     ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);

source/client/src/clientTmq.c

@@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
   SArray* container = &list->container;
   if (src == NULL || src[0] == 0) return -1;
   char* topic = taosStrdup(src);
-  if (topic[0] != '`') {
-    strtolower(topic, src);
-  }
   if (taosArrayPush(container, &topic) == NULL) return -1;
   return 0;
 }
@@ -1243,9 +1240,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pParam);
 
   if (code != 0) {
-    tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
-            vgId, epoch, tstrerror(code), requestId);
-
     if (pMsg->pData) taosMemoryFree(pMsg->pData);
     if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
 
@@ -1267,6 +1261,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
       taosWriteQitem(tmq->mqueue, pRspWrapper);
     } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) {  // poll data while insert
       taosMsleep(500);
+    } else{
+      tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
+               vgId, epoch, tstrerror(code), requestId);
     }
 
     goto CREATE_MSG_FAIL;
@@ -1667,11 +1664,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
     return handleErrorBeforePoll(pVg, pTmq);
   }
 
-  sendInfo->msgInfo = (SDataBuf){
+  sendInfo->msgInfo = (SDataBuf){ .pData = msg, .len = msgSize, .handle = NULL };
-      .pData = msg,
-      .len = msgSize,
-      .handle = NULL,
-  };
 
   sendInfo->requestId = req.reqId;
   sendInfo->requestObjRefId = 0;

source/client/test/smlTest.cpp

@@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) {
   int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(elements.measure, sql);
-  ASSERT_EQ(elements.measureLen, strlen(",st"));
+  ASSERT_EQ(elements.measureLen, strlen("\\,st"));
-  ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3"));
+  ASSERT_EQ(elements.measureEscaped, true);
+  ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3"));
 
   ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
   ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3"));
@@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) {
                         "st,t=1 c=-3.402823466e+39u64 1626006833639000000",
                         "st,t=1 c=-339u64 1626006833639000000",
                         "st,t=1 c=18446744073709551616u64 1626006833639000000",
-                        "st,t=1 c=1=2 1626006833639000000"};
+                        "st,t=1 c=1=2 1626006833639000000,",
+                        // escape error test
+                        // measure comma,space
+                        "s,t,t=1 c=1 1626006833639000000,",
+                        "s t,t=1 c=1 1626006833639000000,",
+                        //tag key comma,equal,space
+                        "st,t,t=1 c=2 1626006833639000000,",
+                        "st,t=t=1 c=2 1626006833639000000,",
+                        "st,t t=1 c=2 1626006833639000000,",
+                        //tag value comma,equal,space
+                        "st,tt=a,a c=2 1626006833639000000,",
+                        "st,t=t=a a c=2 1626006833639000000,",
+                        "st,t t=a=a c=2 1626006833639000000,",
+                        //field key comma,equal,space
+                        "st,tt=aa c,1=2 1626006833639000000,",
+                        "st,tt=aa c=1=2 1626006833639000000,",
+                        "st,tt=aa c 1=2 1626006833639000000,",
+                        //field value    double quote,slash
+                        "st,tt=aa c=\"a\"a\" 1626006833639000000,",
+  };
 
   SSmlHandle *info = smlBuildSmlInfo(NULL);
   info->protocol = TSDB_SML_LINE_PROTOCOL;
@@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) {
   ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
   ASSERT_EQ(kv->keyLen, 5);
   ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
-  ASSERT_EQ(kv->length, 17);
+  ASSERT_EQ(kv->length, 18);
-  ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0);
+  ASSERT_EQ(kv->keyEscaped, true);
+  ASSERT_EQ(kv->valueEscaped, false);
+  ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0);
 
   // nchar
   kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
   ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
   ASSERT_EQ(kv->keyLen, 4);
   ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
-  ASSERT_EQ(kv->length, 8);
+  ASSERT_EQ(kv->length, 9);
-  ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0);
+  ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0);
 
   // bool
   kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);

source/common/src/tglobal.c

@@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsStreamCheckpointTickInterval = 1;
 int32_t tsTtlUnit = 86400;
-int32_t tsTtlPushInterval = 86400;
+int32_t tsTtlPushInterval = 3600;
 int32_t tsGrantHBInterval = 60;
 int32_t tsUptimeInterval = 300;    // seconds
 char    tsUdfdResFuncs[512] = "";  // udfd resident funcs that teardown when udfd exits

source/common/src/tmsg.c

@@ -1409,6 +1409,8 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
   return 0;
 }
 
+void tFreeSAlterUserReq(SAlterUserReq *pReq) { taosMemoryFreeClear(pReq->tagCond); }
+
 int32_t tSerializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -1635,6 +1637,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
       int32_t ref = 0;
       if (tDecodeI32(pDecoder, &ref) < 0) return -1;
       taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref));
+      taosMemoryFree(key);
     }
   }
 
@@ -1831,7 +1834,6 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
     if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1;
   }
 
-
   if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1;
 
   tEndEncode(&encoder);
@@ -1876,7 +1878,6 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
     if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1;
   }
 
-
   if (!tDecodeIsEnd(&decoder)) {
     if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1;
   } else {

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -119,6 +119,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
         pVnode->pFetchQ->threadId);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
 
+  tqNotifyClose(pVnode->pImpl->pTq);
   dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
   while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
 
@@ -141,7 +142,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   dInfo("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (commitAndRemoveWal) {
-    char path[TSDB_FILENAME_LEN] = {0};
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP);
     dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path);
     tfsRmdir(pMgmt->pTfs, path);

source/dnode/mgmt/node_mgmt/src/dmNodes.c

@@ -132,11 +132,15 @@ int32_t dmRunDnode(SDnode *pDnode) {
   int32_t count = 0;
   if (dmOpenNodes(pDnode) != 0) {
     dError("failed to open nodes since %s", terrstr());
+    dmCloseNodes(pDnode);
     return -1;
   }
 
   if (dmStartNodes(pDnode) != 0) {
     dError("failed to start nodes since %s", terrstr());
+    dmSetStatus(pDnode, DND_STAT_STOPPED);
+    dmStopNodes(pDnode);
+    dmCloseNodes(pDnode);
     return -1;
   }
 

source/dnode/mnode/impl/src/mndDb.c

@@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
   if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
   if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
+  if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1;
+  if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1;
   if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
   if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
 
@@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
   if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
   if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
-  if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
-  if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
   if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
 }
 
@@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
     int32_t dbLen = strlen(dbObj.name) + 1;
     mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
     dbObj.cfg.hashPrefix += dbLen;
+  } else if (dbObj.cfg.hashPrefix < 0) {
+    int32_t dbLen = strlen(dbObj.name) + 1;
+    mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen);
+    dbObj.cfg.hashPrefix -= dbLen;
   }
 
   SVgObj *pVgroups = NULL;
@@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
     int16_t hashPrefix = pDb->cfg.hashPrefix;
     if (hashPrefix > 0) {
       hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
+    } else if (hashPrefix < 0) {
+      hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1;
     }
     colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);
 

source/dnode/mnode/impl/src/mndMain.c

@@ -124,11 +124,7 @@ static void mndCalMqRebalance(SMnode *pMnode) {
   int32_t contLen = 0;
   void   *pReq = mndBuildTimerMsg(&contLen);
   if (pReq != NULL) {
-    SRpcMsg rpcMsg = {
+    SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen };
-        .msgType = TDMT_MND_TMQ_TIMER,
-        .pCont = pReq,
-        .contLen = contLen,
-    };
     tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
   }
 }

source/dnode/mnode/impl/src/mndUser.c

@@ -390,6 +390,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
       SDB_GET_INT32(pRaw, dataPos, &ref, _OVER);
 
       taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref));
+      taosMemoryFree(key);
     }
   }
 
@@ -956,6 +957,7 @@ _OVER:
     mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
   }
 
+  tFreeSAlterUserReq(&alterReq);
   mndReleaseUser(pMnode, pOperUser);
   mndReleaseUser(pMnode, pUser);
   mndUserFreeObj(&newUser);

source/dnode/vnode/inc/vnode.h

@@ -162,7 +162,6 @@ int32_t     metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
 #endif
 
 // tsdb
-// typedef struct STsdb STsdb;
 typedef struct STsdbReader STsdbReader;
 
 #define TSDB_DEFAULT_STT_FILE  8

source/dnode/vnode/src/inc/tq.h

@@ -180,16 +180,9 @@ int32_t tqStreamTasksScanWal(STQ* pTq);
 
 // tq util
 char*   createStreamTaskIdStr(int64_t streamId, int32_t taskId);
-void    createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId);
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
-int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
 int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
 
-void    doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
-void    saveOffsetForAllTasks(STQ* pTq, int64_t ver);
-void    initOffsetForAllRestoreTasks(STQ* pTq);
-int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/tsdb.h

@@ -16,6 +16,7 @@
 #ifndef _TD_VNODE_TSDB_H_
 #define _TD_VNODE_TSDB_H_
 
+#include "tsimplehash.h"
 #include "vnodeInt.h"
 
 #ifdef __cplusplus
@@ -125,11 +126,13 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter);
 // SRowMerger
 int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema);
 int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
-
-// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
 void tsdbRowMergerClear(SRowMerger *pMerger);
-// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
 int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow);
+
+int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema);
+void tsdbRowMergerClear_rv(SRowMerger* pMerger);
+void tsdbRowMergerCleanup_rv(SRowMerger* pMerger);
+
 // TABLEID
 int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
 // TSDBKEY
@@ -224,7 +227,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward,
 void   *tsdbTbDataIterDestroy(STbDataIter *pIter);
 void    tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
 bool    tsdbTbDataIterNext(STbDataIter *pIter);
-void    tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum);
+void    tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum);
 
 // STbData
 int32_t tsdbGetNRowsInTbData(STbData *pTbData);
@@ -324,6 +327,7 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData,
 
 #define TSDB_FILTER_FLAG_BY_VERSION           0x1
 #define TSDB_FILTER_FLAG_BY_TABLEID           0x2
+#define TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE 0x4
 
 #define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn)))
 /* open */
@@ -705,7 +709,6 @@ typedef struct SSttBlockLoadInfo {
 typedef struct SMergeTree {
   int8_t             backward;
   SRBTree            rbt;
-  SArray            *pIterList;
   SLDataIter        *pIter;
   bool               destroyLoadInfo;
   SSttBlockLoadInfo *pLoadInfo;
@@ -751,13 +754,29 @@ struct SDiskDataBuilder {
   SBlkInfo     bi;
 };
 
+typedef struct SLDataIter {
+  SRBTreeNode        node;
+  SSttBlk           *pSttBlk;
+  SDataFReader      *pReader;
+  int32_t            iStt;
+  int8_t             backward;
+  int32_t            iSttBlk;
+  int32_t            iRow;
+  SRowInfo           rInfo;
+  uint64_t           uid;
+  STimeWindow        timeWindow;
+  SVersionRange      verRange;
+  SSttBlockLoadInfo *pBlockLoadInfo;
+  bool               ignoreEarlierTs;
+} SLDataIter;
+
+#define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row))
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
                        STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
-                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange);
+                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter);
 void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool    tMergeTreeNext(SMergeTree *pMTree);
 bool    tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
-TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);
 
 SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt);
@@ -782,6 +801,7 @@ typedef struct SCacheRowsReader {
   STableKeyInfo     *pTableList;  // table id list
   int32_t            numOfTables;
   SSttBlockLoadInfo *pLoadInfo;
+  SLDataIter        *pDataIter;
   STsdbReadSnap     *pReadSnap;
   SDataFReader      *pDataFReader;
   SDataFReader      *pDataFReaderLast;

source/dnode/vnode/src/inc/vnodeInt.h

@@ -190,6 +190,7 @@ int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
 int  tqInit();
 void tqCleanUp();
 STQ* tqOpen(const char* path, SVnode* pVnode);
+void tqNotifyClose(STQ*);
 void tqClose(STQ*);
 int  tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int  tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -187,23 +187,24 @@ _err:
 
 int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t    code = 0;
+  int32_t    line = 0;
   SMeta*     pMeta = pWriter->pMeta;
   SMetaEntry metaEntry = {0};
   SDecoder*  pDecoder = &(SDecoder){0};
 
   tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
   code = metaDecodeEntry(pDecoder, &metaEntry);
-  if (code) goto _err;
+  VND_CHECK_CODE(code, line, _err);
 
   code = metaHandleEntry(pMeta, &metaEntry);
-  if (code) goto _err;
+  VND_CHECK_CODE(code, line, _err);
 
   tDecoderClear(pDecoder);
   return code;
 
 _err:
   tDecoderClear(pDecoder);
-  metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
   return code;
 }
 

source/dnode/vnode/src/meta/metaTable.c

@@ -936,8 +936,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     int   tLen = 0;
 
     if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
-      version = ((SUidIdxVal *)tData)[0].version;
+      STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version};
-      STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version};
       if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
         SDecoder   tdc = {0};
         SMetaEntry stbEntry = {0};
@@ -1029,7 +1028,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
   metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode),
             pME->version, pME->uid, ctimeKey.ctime);
 
-  return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
 }
 
 int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -1044,7 +1043,7 @@ int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
   if (metaBuildNColIdxKey(&ncolKey, pME) < 0) {
     return 0;
   }
-  return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
 }
 
 int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -1878,24 +1877,24 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
 }
 
 static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
 }
 
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
+  return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
 }
 
 static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   STtlIdxKey ttlKey = {0};
   metaBuildTtlIdxKey(&ttlKey, pME);
   if (ttlKey.dtime == 0) return 0;
-  return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
 
-  return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
+  return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
                      ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn);
 }
 
@@ -2065,49 +2064,66 @@ _exit:
 }
 
 int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+  int32_t code = 0;
+  int32_t line = 0;
   metaWLock(pMeta);
 
   // save to table.db
-  if (metaSaveToTbDb(pMeta, pME) < 0) goto _err;
+  code = metaSaveToTbDb(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   // update uid.idx
-  if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
+  code = metaUpdateUidIdx(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   // update name.idx
-  if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err;
+  code = metaUpdateNameIdx(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   if (pME->type == TSDB_CHILD_TABLE) {
     // update ctb.idx
-    if (metaUpdateCtbIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateCtbIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
 
     // update tag.idx
-    if (metaUpdateTagIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateTagIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
   } else {
     // update schema.db
-    if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err;
+    code = metaSaveToSkmDb(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
 
     if (pME->type == TSDB_SUPER_TABLE) {
-      if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err;
+      code = metaUpdateSuidIdx(pMeta, pME);
+      VND_CHECK_CODE(code, line, _err);
     }
   }
 
-  if (metaUpdateCtimeIdx(pMeta, pME) < 0) goto _err;
+  code = metaUpdateCtimeIdx(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   if (pME->type == TSDB_NORMAL_TABLE) {
-    if (metaUpdateNcolIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateNcolIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
   }
 
   if (pME->type != TSDB_SUPER_TABLE) {
-    if (metaUpdateTtlIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateTtlIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
   }
 
   metaULock(pMeta);
+  metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode),
+            pME->version, pME->uid, pME->name);
   return 0;
 
 _err:
   metaULock(pMeta);
+  metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s",
+            TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name);
   return -1;
 }
+
 // refactor later
 void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
 void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }

source/dnode/vnode/src/tq/tq.c

@@ -154,6 +154,31 @@ void tqClose(STQ* pTq) {
   taosMemoryFree(pTq);
 }
 
+void tqNotifyClose(STQ* pTq) {
+  if (pTq != NULL) {
+    taosWLockLatch(&pTq->pStreamMeta->lock);
+
+    void* pIter = NULL;
+    while (1) {
+      pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
+      if (pIter == NULL) {
+        break;
+      }
+
+      SStreamTask* pTask = *(SStreamTask**)pIter;
+      tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr);
+      pTask->status.taskStatus = TASK_STATUS__STOP;
+
+      int64_t st = taosGetTimestampMs();
+      qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
+      int64_t el = taosGetTimestampMs() - st;
+      tqDebug("vgId:%d s-task:%s is closed in %" PRId64 "ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el);
+    }
+
+    taosWUnLockLatch(&pTq->pStreamMeta->lock);
+  }
+}
+
 static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch,
                              int64_t consumerId, int32_t type) {
   int32_t len = 0;
@@ -573,6 +598,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->pMsgCb = &pTq->pVnode->msgCb;
   pTask->pMeta = pTq->pStreamMeta;
   pTask->chkInfo.version = ver;
+  pTask->chkInfo.currentVer = ver;
 
   // expand executor
   if (pTask->fillHistory) {
@@ -966,14 +992,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   int32_t* pRef = taosMemoryMalloc(sizeof(int32_t));
   *pRef = 1;
 
+  taosWLockLatch(&pTq->pStreamMeta->lock);
+
   void* pIter = NULL;
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
-    SStreamTask* pTask = *(SStreamTask**)pIter;
+      break;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue;
+    }
 
-    qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver);
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      continue;
+    }
+
+    qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver);
 
     if (!failed) {
       SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
@@ -983,15 +1016,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
       atomic_add_fetch_32(pRefBlock->dataRef, 1);
 
       if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
-        qError("stream task input del failed, task id %d", pTask->id.taskId);
-
         atomic_sub_fetch_32(pRef, 1);
         taosFreeQitem(pRefBlock);
         continue;
       }
 
       if (streamSchedExec(pTask) < 0) {
-        qError("stream task launch failed, task id %d", pTask->id.taskId);
+        qError("s-task:%s stream task launch failed", pTask->id.idStr);
         continue;
       }
 
@@ -1000,8 +1031,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
     }
   }
 
+  taosWUnLockLatch(&pTq->pStreamMeta->lock);
+
   int32_t ref = atomic_sub_fetch_32(pRef, 1);
-  /*A(ref >= 0);*/
   if (ref == 0) {
     blockDataDestroy(pDelBlock);
     taosMemoryFree(pRef);
@@ -1032,23 +1064,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   }
   blockDataDestroy(pDelBlock);
 #endif
-
   return 0;
 }
 
-static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
-                                         const char* key, int64_t ver) {
-  doSaveTaskOffset(pOffsetStore, key, ver);
-  int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
-
-  // remove the offset, if all functions are completed successfully.
-  if (code == TSDB_CODE_SUCCESS) {
-    tqOffsetDelete(pOffsetStore, key);
-  }
-
-  return code;
-}
-
 int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
 #if 0
   void* pIter = NULL;
@@ -1309,9 +1327,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {
     return -1;
   }
 
-  tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
+  tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
-  initOffsetForAllRestoreTasks(pTq);
-
   pRunReq->head.vgId = vgId;
   pRunReq->streamId = 0;
   pRunReq->taskId = WAL_READ_TASKS_ID;

source/dnode/vnode/src/tq/tqRead.c

@@ -1023,6 +1023,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
   }
 
   // update the table list handle for each stream scanner/wal reader
+  taosWLockLatch(&pTq->pStreamMeta->lock);
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
@@ -1039,5 +1040,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
     }
   }
 
+  taosWUnLockLatch(&pTq->pStreamMeta->lock);
+
   return 0;
 }

source/dnode/vnode/src/tq/tqRestore.c

@@ -15,13 +15,12 @@
 
 #include "tq.h"
 
-static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle);
+static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle);
-static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
 
 // this function should be executed by stream threads.
 // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
 // will not stop eventually.
-int tqStreamTasksScanWal(STQ* pTq) {
+int32_t tqStreamTasksScanWal(STQ* pTq) {
   int32_t vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
   int64_t st = taosGetTimestampMs();
@@ -32,7 +31,7 @@ int tqStreamTasksScanWal(STQ* pTq) {
 
     // check all restore tasks
     bool shouldIdle = true;
-    streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle);
+    createStreamRunReq(pTq->pStreamMeta, &shouldIdle);
 
     int32_t times = 0;
 
@@ -55,42 +54,14 @@ int tqStreamTasksScanWal(STQ* pTq) {
 
   int64_t el = (taosGetTimestampMs() - st);
   tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%"PRId64" ms", vgId, el);
-
-  // restore wal scan flag
-//  atomic_store_8(&pTq->pStreamMeta->walScan, 0);
   return 0;
 }
 
-//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
+static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) {
-//  int32_t numOfTask = taosArrayGetSize(pTaskList);
+  SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t));
-//  if (numOfTask <= 0)  {
-//    return TSDB_CODE_SUCCESS;
-//  }
-//
-//  // todo: add lock
-//  for (int32_t i = 0; i < numOfTask; ++i) {
-//    SStreamTask* pTask = taosArrayGetP(pTaskList, i);
-//    tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
-//            pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
-//    taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
-//
-//    // NOTE: do not change the following order
-//    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
-//    taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
-//  }
-//
-//  return TSDB_CODE_SUCCESS;
-//}
-
-int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) {
   void*   pIter = NULL;
-  int32_t vgId = pStreamMeta->vgId;
-
-  *pScanIdle = true;
-
-  bool allWalChecked = true;
-  tqDebug("vgId:%d start to check wal to extract new submit block", vgId);
 
+  taosWLockLatch(&pStreamMeta->lock);
   while(1) {
     pIter = taosHashIterate(pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
@@ -98,44 +69,71 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto
     }
 
     SStreamTask* pTask = *(SStreamTask**)pIter;
+    taosArrayPush(pTaskIdList, &pTask->id.taskId);
+  }
+
+  taosWUnLockLatch(&pStreamMeta->lock);
+  return pTaskIdList;
+}
+
+int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
+  *pScanIdle = true;
+  bool    noNewDataInWal = true;
+  int32_t vgId = pStreamMeta->vgId;
+
+  int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks);
+  if (numOfTasks == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks);
+  SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks);
+
+  // update the new task number
+  numOfTasks = taosArrayGetSize(pTaskIdList);
+  for (int32_t i = 0; i < numOfTasks; ++i) {
+    int32_t*     pTaskId = taosArrayGet(pTaskIdList, i);
+    SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId);
+    if (pTask == NULL) {
+      continue;
+    }
+
+    int32_t status = pTask->status.taskStatus;
     if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      tqDebug("s-task:%s not source task, no need to start", pTask->id.idStr);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE ||
+    if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE ||
-        pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
+        status == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
+      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status);
-              pTask->status.taskStatus);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
-    // check if offset value exists
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
     if (tInputQueueIsFull(pTask)) {
       tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
     *pScanIdle = false;
 
-    // check if offset value exists
-    STqOffset* pOffset = tqOffsetRead(pOffsetStore, key);
-    ASSERT(pOffset != NULL);
-
     // seek the stored version and extract data from WAL
-    int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version);
+    int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
     if (code != TSDB_CODE_SUCCESS) {  // no data in wal, quit
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
     // append the data for the stream
-    tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr);
+    tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer);
 
     SPackedData packData = {0};
     code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData);
     if (code != TSDB_CODE_SUCCESS) {  // failed, continue
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
@@ -143,28 +141,32 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto
     if (p == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
-    allWalChecked = false;
+    noNewDataInWal = false;
 
-    tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr);
     code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver);
     if (code == TSDB_CODE_SUCCESS) {
-      pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader);
+      pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
       tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
-              pOffset->val.version);
+              pTask->chkInfo.currentVer);
     } else {
-      // do nothing
+      tqError("s-task:%s append input queue failed, ver:%"PRId64, pTask->id.idStr, pTask->chkInfo.currentVer);
     }
 
     streamDataSubmitDestroy(p);
     taosFreeQitem(p);
+    streamMetaReleaseTask(pStreamMeta, pTask);
   }
 
-  if (allWalChecked) {
+  // all wal are checked, and no new data available in wal.
+  if (noNewDataInWal) {
     *pScanIdle = true;
   }
+
+  taosArrayDestroy(pTaskIdList);
   return 0;
 }
 

source/dnode/vnode/src/tq/tqUtil.c

@@ -25,21 +25,6 @@ char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) {
   return taosStrdup(buf);
 }
 
-// stream_task:stream_id:task_id
-void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
-  int32_t n = 12;
-  char* p = dst;
-
-  memcpy(p, "stream_task:", n);
-  p += n;
-
-  int32_t inc = tintToHex(streamId, p);
-  p += inc;
-
-  *(p++) = ':';
-  tintToHex(taskId, p);
-}
-
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
   int32_t code = tAppendDataToInputQueue(pTask, pQueueItem);
   if (code < 0) {
@@ -55,75 +40,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI
   return TSDB_CODE_SUCCESS;
 }
 
-void initOffsetForAllRestoreTasks(STQ* pTq) {
-  void* pIter = NULL;
-
-  while(1) {
-    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) {
-      break;
-    }
-
-    SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
-      continue;
-    }
-
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus);
-      continue;
-    }
-
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-    if (pOffset == NULL) {
-      doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version);
-    }
-  }
-}
-
-void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
-  void* pIter = NULL;
-
-  while(1) {
-    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) {
-      break;
-    }
-
-    SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
-      continue;
-    }
-
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
-              pTask->status.taskStatus);
-      continue;
-    }
-
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-    if (pOffset == NULL) {
-      doSaveTaskOffset(pTq->pOffsetStore, key, ver);
-    }
-  }
-}
-
-void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
-  STqOffset offset = {0};
-  tqOffsetResetToLog(&offset.val, ver);
-
-  tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
-
-  // keep the offset info in the offset store
-  tqOffsetWrite(pOffsetStore, &offset);
-}
-
 static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
   pRsp->reqOffset = pReq->reqOffset;
 

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -598,6 +598,7 @@ typedef struct {
   SMergeTree         mergeTree;
   SMergeTree        *pMergeTree;
   SSttBlockLoadInfo *pLoadInfo;
+  SLDataIter*        pDataIter;
   int64_t            lastTs;
 } SFSLastNextRowIter;
 
@@ -645,7 +646,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
       }
       tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
-                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
+                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, state->pDataIter);
       state->pMergeTree = &state->mergeTree;
       state->state = SFSLASTNEXTROW_BLOCKROW;
     }
@@ -667,7 +668,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
         state->state = SFSLASTNEXTROW_FILESET;
         goto _next_fileset;
       }
-      state->row = tMergeTreeGetRow(&state->mergeTree);
+      state->row = *tMergeTreeGetRow(&state->mergeTree);
       *ppRow = &state->row;
 
       if (TSDBROW_TS(&state->row) <= state->lastTs) {
@@ -1211,7 +1212,7 @@ typedef struct {
 } CacheNextRowIter;
 
 static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
-                               SSttBlockLoadInfo *pLoadInfo, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader,
+                               SSttBlockLoadInfo *pLoadInfo, SLDataIter* pLDataIter, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader,
                                SDataFReader **pDataFReaderLast, int64_t lastTs) {
   int code = 0;
 
@@ -1274,6 +1275,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
   pIter->fsLastState.pLoadInfo = pLoadInfo;
   pIter->fsLastState.pDataFReader = pDataFReaderLast;
   pIter->fsLastState.lastTs = lastTs;
+  pIter->fsLastState.pDataIter = pLDataIter;
 
   pIter->fsState.state = SFSNEXTROW_FS;
   pIter->fsState.pTsdb = pTsdb;
@@ -1465,7 +1467,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
-  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader,
+  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader,
                   &pr->pDataFReaderLast, pr->lastTs);
 
   do {
@@ -1622,7 +1624,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
-  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader,
+  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader,
                   &pr->pDataFReaderLast, pr->lastTs);
 
   do {

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -187,13 +187,21 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
     }
   }
 
-  int32_t numOfStt = ((SVnode*)pVnode)->config.sttTrigger;
+  SVnodeCfg* pCfg = &((SVnode*)pVnode)->config;
+
+  int32_t numOfStt = pCfg->sttTrigger;
   p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt);
   if (p->pLoadInfo == NULL) {
     tsdbCacherowsReaderClose(p);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter));
+  if (p->pDataIter == NULL) {
+    tsdbCacherowsReaderClose(p);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
   p->idstr = taosStrdup(idstr);
   taosThreadMutexInit(&p->readerMutex, NULL);
 
@@ -215,6 +223,7 @@ void* tsdbCacherowsReaderClose(void* pReader) {
     taosMemoryFree(p->pSchema);
   }
 
+  taosMemoryFreeClear(p->pDataIter);
   taosMemoryFree(p->pCurrSchema);
 
   destroyLastBlockLoadInfo(p->pLoadInfo);

source/dnode/vnode/src/tsdb/tsdbDataIter.c

@@ -14,6 +14,7 @@
  */
 
 #include "tsdb.h"
+#include "vnodeInt.h"
 
 // STsdbDataIter2
 /* open */
@@ -202,13 +203,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
   for (;;) {
     while (pIter->dIter.iRow < pIter->dIter.bData.nRow) {
       if (pFilterInfo) {
-        if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
-          if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) {
-            pIter->dIter.iRow = pIter->dIter.bData.nRow;
-            continue;
-          }
-        }
-
         if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
           if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver ||
               pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) {
@@ -232,13 +226,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
 
         // filter
         if (pFilterInfo) {
-          if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
-            if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) {
-              pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem;
-              continue;
-            }
-          }
-
           if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
             if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) {
               pIter->dIter.iDataBlk++;
@@ -262,7 +249,8 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
         if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) {
           SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx);
 
-          if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) {
+          if (pFilterInfo) {
+            if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
               int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid);
               if (c == 0) {
                 pIter->dIter.iBlockIdx++;
@@ -272,6 +260,15 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
               }
             }
 
+            if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
+              SMetaInfo info;
+              if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pBlockIdx->uid, &info, NULL)) {
+                pIter->dIter.iBlockIdx++;
+                continue;
+              }
+            }
+          }
+
           code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk);
           TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -304,14 +301,24 @@ static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* p
   for (;;) {
     while (pIter->sIter.iRow < pIter->sIter.bData.nRow) {
       if (pFilterInfo) {
-        if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
         int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
+        if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
           if (pFilterInfo->tbid.uid == uid) {
             pIter->sIter.iRow++;
             continue;
           }
         }
 
+        if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
+          if (pIter->rowInfo.uid != uid) {
+            SMetaInfo info;
+            if (metaGetInfo(pIter->sIter.pReader->pTsdb->pVnode->pMeta, uid, &info, NULL)) {
+              pIter->sIter.iRow++;
+              continue;
+            }
+          }
+        }
+
         if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
           if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] ||
               pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) {
@@ -395,6 +402,16 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
       if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) {
         SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx);
 
+        if (pFilterInfo) {
+          if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
+            SMetaInfo info;
+            if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pDelIdx->uid, &info, NULL)) {
+              pIter->tIter.iDelIdx++;
+              continue;
+            }
+          }
+        }
+
         code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData);
         TSDB_CHECK_CODE(code, lino, _exit);
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <util/tsimplehash.h>
 #include "tsdb.h"
 
 #define MEM_MIN_HASH 1024
@@ -298,12 +299,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) {
   return rowsNum;
 }
 
-void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj*        pTableMap, int64_t *rowsNum) {
+void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj* pTableMap, int64_t *rowsNum) {
   taosRLockLatch(&pMemTable->latch);
   for (int32_t i = 0; i < pMemTable->nBucket; ++i) {
     STbData *pTbData = pMemTable->aBucket[i];
     while (pTbData) {
-      void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
+      void* p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
       if (p == NULL) {
         pTbData = pTbData->next;
         continue;

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -16,22 +16,6 @@
 #include "tsdb.h"
 
 // SLDataIter =================================================
-struct SLDataIter {
-  SRBTreeNode        node;
-  SSttBlk           *pSttBlk;
-  SDataFReader      *pReader;
-  int32_t            iStt;
-  int8_t             backward;
-  int32_t            iSttBlk;
-  int32_t            iRow;
-  SRowInfo           rInfo;
-  uint64_t           uid;
-  STimeWindow        timeWindow;
-  SVersionRange      verRange;
-  SSttBlockLoadInfo *pBlockLoadInfo;
-  bool               ignoreEarlierTs;
-};
-
 SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols,
                                             int32_t numOfSttTrigger) {
   SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo));
@@ -88,6 +72,10 @@ void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double
 }
 
 void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
+  if (pLoadInfo == NULL) {
+    return NULL;
+  }
+
   for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) {
     pLoadInfo[i].currentLoadBlockIndex = 1;
     pLoadInfo[i].blockIndex[0] = -1;
@@ -264,25 +252,21 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint
   }
 }
 
-int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
+int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
                        uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo,
                        const char *idStr, bool strictTimeRange) {
   int32_t code = TSDB_CODE_SUCCESS;
 
-  *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
+  pIter->uid = uid;
-  if (*pIter == NULL) {
+  pIter->pReader = pReader;
-    code = TSDB_CODE_OUT_OF_MEMORY;
+  pIter->iStt = iStt;
-    goto _exit;
+  pIter->backward = backward;
-  }
+  pIter->verRange.minVer = pRange->minVer;
+  pIter->verRange.maxVer = pRange->maxVer;
+  pIter->timeWindow.skey = pTimeWindow->skey;
+  pIter->timeWindow.ekey = pTimeWindow->ekey;
 
-  (*pIter)->uid = uid;
+  pIter->pBlockLoadInfo = pBlockLoadInfo;
-  (*pIter)->pReader = pReader;
-  (*pIter)->iStt = iStt;
-  (*pIter)->backward = backward;
-  (*pIter)->verRange = *pRange;
-  (*pIter)->timeWindow = *pTimeWindow;
-
-  (*pIter)->pBlockLoadInfo = pBlockLoadInfo;
 
   if (!pBlockLoadInfo->sttBlockLoaded) {
     int64_t st = taosGetTimestampUs();
@@ -290,7 +274,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
-      goto _exit;
+      return code;
     }
 
     // only apply to the child tables, ordinary tables will not incur this filter procedure.
@@ -306,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
           // no qualified stt block existed
           taosArrayClear(pBlockLoadInfo->aSttBlk);
 
-          (*pIter)->iSttBlk = -1;
+          pIter->iSttBlk = -1;
           double el = (taosGetTimestampUs() - st) / 1000.0;
           tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
           return code;
@@ -339,31 +323,27 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
-  (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
+  pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
-  if ((*pIter)->iSttBlk != -1) {
+  if (pIter->iSttBlk != -1) {
-    (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
+    pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
-    (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1;
+    pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1;
 
-    if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) ||
+    if ((!backward) && ((strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) ||
-                        (!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) {
+                        (!strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) {
-      (*pIter)->pSttBlk = NULL;
+      pIter->pSttBlk = NULL;
     }
 
-    if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) ||
+    if (backward && ((strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) ||
-                     (!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) {
+                     (!strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) {
-      (*pIter)->pSttBlk = NULL;
+      pIter->pSttBlk = NULL;
-      (*pIter)->ignoreEarlierTs = true;
+      pIter->ignoreEarlierTs = true;
     }
   }
 
   return code;
-
-_exit:
-  taosMemoryFree(*pIter);
-  return code;
 }
 
-void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); }
+void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */}
 
 void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) {
   int32_t step = pIter->backward ? -1 : 1;
@@ -590,43 +570,38 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR
 
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
                        STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
-                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange) {
+                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
   pMTree->backward = backward;
   pMTree->pIter = NULL;
-  pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
-  if (pMTree->pIterList == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
   pMTree->idStr = idStr;
+
   if (!pMTree->backward) {  // asc
     tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
   } else {  // desc
     tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn);
   }
-  int32_t code = TSDB_CODE_SUCCESS;
 
   pMTree->pLoadInfo = pBlockLoadInfo;
   pMTree->destroyLoadInfo = destroyLoadInfo;
   pMTree->ignoreEarlierTs = false;
 
   for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) {  // open all last file
-    struct SLDataIter *pIter = NULL;
+    memset(&pLDataIter[i], 0, sizeof(SLDataIter));
-    code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
+    code = tLDataIterOpen(&pLDataIter[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
                           &pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
 
-    bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
+    bool hasVal = tLDataIterNextRow(&pLDataIter[i], pMTree->idStr);
     if (hasVal) {
-      taosArrayPush(pMTree->pIterList, &pIter);
+      tMergeTreeAddIter(pMTree, &pLDataIter[i]);
-      tMergeTreeAddIter(pMTree, pIter);
     } else {
       if (!pMTree->ignoreEarlierTs) {
-        pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs;
+        pMTree->ignoreEarlierTs = pLDataIter[i].ignoreEarlierTs;
       }
-      tLDataIterClose(pIter);
     }
   }
 
@@ -674,18 +649,8 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
   return pMTree->pIter != NULL;
 }
 
-TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; }
-
 void tMergeTreeClose(SMergeTree *pMTree) {
-  size_t size = taosArrayGetSize(pMTree->pIterList);
-  for (int32_t i = 0; i < size; ++i) {
-    SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i);
-    tLDataIterClose(pIter);
-  }
-
-  pMTree->pIterList = taosArrayDestroy(pMTree->pIterList);
   pMTree->pIter = NULL;
-
   if (pMTree->destroyLoadInfo) {
     pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo);
     pMTree->destroyLoadInfo = false;

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -70,8 +70,9 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) {
 
   if (pReader->pIter) {
     // iter to next with filter info (sver, ever)
-    code = tsdbDataIterNext2(pReader->pIter,
+    code = tsdbDataIterNext2(
-                             &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION,  // flag
+        pReader->pIter,
+        &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,  // flag
                            .sver = pReader->sver,
                            .ever = pReader->ever});
     TSDB_CHECK_CODE(code, lino, _exit);
@@ -94,8 +95,9 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) {
 
     if (pReader->pIter) {
       // iter to valid row
-      code = tsdbDataIterNext2(pReader->pIter,
+      code = tsdbDataIterNext2(
-                               &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION,  // flag
+          pReader->pIter,
+          &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,  // flag
                              .sver = pReader->sver,
                              .ever = pReader->ever});
       TSDB_CHECK_CODE(code, lino, _exit);
@@ -139,7 +141,8 @@ static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInf
   int32_t lino = 0;
 
   if (pReader->pIter) {
-    code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION,  // flag
+    code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION |
+                                                                        TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,  // flag
                                                                 .sver = pReader->sver,
                                                                 .ever = pReader->ever});
     TSDB_CHECK_CODE(code, lino, _exit);
@@ -346,8 +349,9 @@ static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppD
   int32_t lino = 0;
 
   code = tsdbDataIterNext2(
-      pReader->pTIter,
+      pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,
-      &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, .sver = pReader->sver, .ever = pReader->ever});
+                                          .sver = pReader->sver,
+                                          .ever = pReader->ever});
   TSDB_CHECK_CODE(code, lino, _exit);
 
   if (ppDelInfo) {

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -719,6 +719,62 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
   if (NULL == pTSchema) {
     pTSchema = pMerger->pTSchema;
   }
+
+  if (taosArrayGetSize(pMerger->pArray) == 0) {
+    // ts
+    jCol = 0;
+    pTColumn = &pTSchema->columns[jCol++];
+
+    ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
+
+    *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
+    if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      return code;
+      //      goto _exit;
+    }
+
+    // other
+    for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pMerger->pTSchema->numOfCols; ++iCol) {
+      pTColumn = &pMerger->pTSchema->columns[iCol];
+      if (pTSchema->columns[jCol].colId < pTColumn->colId) {
+        ++jCol;
+        --iCol;
+        continue;
+      } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
+        taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
+        continue;
+      }
+
+      tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
+      if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+        uint8_t *pVal = pColVal->value.pData;
+
+        pColVal->value.pData = NULL;
+        code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
+        if (code) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+
+        if (pColVal->value.nData) {
+          memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+        }
+      }
+
+      if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        return code;
+      }
+    }
+
+    for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) {
+      pTColumn = &pMerger->pTSchema->columns[iCol];
+      taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
+    }
+
+    pMerger->version = key.version;
+    return 0;
+  } else {
     ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
 
     for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
@@ -778,58 +834,41 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
     pMerger->version = key.version;
     return code;
   }
-/*
+}
-int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
-  int32_t   code = 0;
-  TSDBKEY   key = TSDBROW_KEY(pRow);
-  SColVal  *pColVal = &(SColVal){0};
-  STColumn *pTColumn;
 
-  pMerger->pTSchema = pTSchema;
+int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) {
-  pMerger->version = key.version;
+  pMerger->pTSchema = pSchema;
-
+  pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal));
-  pMerger->pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal));
   if (pMerger->pArray == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
+  } else {
-  }
+    return TSDB_CODE_SUCCESS;
-
-  // ts
-  pTColumn = &pTSchema->columns[0];
-
-  ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
-
-  *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
-  if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
-  }
-
-  // other
-  for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
-    tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
-    if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
-      uint8_t *pVal = pColVal->value.pData;
-
-      pColVal->value.pData = NULL;
-      code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
-      if (code) goto _exit;
-
-      if (pColVal->value.nData) {
-        memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
   }
 }
 
-    if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
+void tsdbRowMergerClear_rv(SRowMerger* pMerger) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
+  for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
-      goto _exit;
+    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+      tFree(pTColVal->value.pData);
     }
   }
 
-_exit:
+  taosArrayClear(pMerger->pArray);
-  return code;
 }
-*/
+
+void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) {
+  int32_t numOfCols = taosArrayGetSize(pMerger->pArray);
+  for (int32_t iCol = 1; iCol < numOfCols; iCol++) {
+    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+      tFree(pTColVal->value.pData);
+    }
+  }
+
+  taosArrayDestroy(pMerger->pArray);
+}
+
 void tsdbRowMergerClear(SRowMerger *pMerger) {
   for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
     SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
@@ -1218,9 +1257,10 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
     int32_t   iColumn = 1;
     STColumn *pTColumn = &pTSchema->columns[iColumn];
     for (int32_t iCid = 0; iCid < nCid; iCid++) {
-      if (ASSERTS(pTColumn != NULL, "invalid input param")) {
+
-        code = TSDB_CODE_INVALID_PARA;
+      // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer
-        goto _exit;
+      if (pTColumn == NULL) {
+        continue;
       }
 
       while (pTColumn->colId < aCid[iCid]) {
@@ -1229,9 +1269,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
         pTColumn = &pTSchema->columns[iColumn];
       }
 
-      if (ASSERTS(pTColumn->colId == aCid[iCid], "invalid input param")) {
+      if (pTColumn->colId != aCid[iCid]) {
-        code = TSDB_CODE_INVALID_PARA;
+        continue;
-        goto _exit;
       }
 
       tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type,

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -1348,6 +1348,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
           code = terrno;
           goto _exit;
         }
+        terrno = 0;
         pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid;  // update uid if table exist for using below
       }
     }

source/libs/catalog/inc/catalogInt.h

@@ -58,6 +58,7 @@ typedef enum {
   CTG_CI_OTHERTABLE_META,
   CTG_CI_TBL_SMA,
   CTG_CI_TBL_CFG,
+  CTG_CI_TBL_TAG,
   CTG_CI_INDEX_INFO,
   CTG_CI_USER,
   CTG_CI_UDF,
@@ -110,6 +111,7 @@ typedef enum {
   CTG_TASK_GET_SVR_VER,
   CTG_TASK_GET_TB_META_BATCH,
   CTG_TASK_GET_TB_HASH_BATCH,
+  CTG_TASK_GET_TB_TAG,
 } CTG_TASK_TYPE;
 
 typedef enum {
@@ -152,6 +154,11 @@ typedef struct SCtgTbCacheInfo {
   int32_t  tbType;
 } SCtgTbCacheInfo;
 
+typedef struct SCtgTbMetaParam {
+  SName*  pName;
+  int32_t flag;
+} SCtgTbMetaParam;
+
 typedef struct SCtgTbMetaCtx {
   SCtgTbCacheInfo tbInfo;
   int32_t         vgId;
@@ -186,6 +193,11 @@ typedef struct SCtgTbCfgCtx {
   SVgroupInfo* pVgInfo;
 } SCtgTbCfgCtx;
 
+typedef struct SCtgTbTagCtx {
+  SName*       pName;
+  SVgroupInfo* pVgInfo;
+} SCtgTbTagCtx;
+
 typedef struct SCtgDbVgCtx {
   char dbFName[TSDB_DB_FNAME_LEN];
 } SCtgDbVgCtx;
@@ -304,6 +316,7 @@ typedef struct SCtgJob {
   catalogCallback  userFp;
   int32_t          tbMetaNum;
   int32_t          tbHashNum;
+  int32_t          tbTagNum;
   int32_t          dbVgNum;
   int32_t          udfNum;
   int32_t          qnodeNum;
@@ -346,6 +359,7 @@ typedef struct SCtgSubRes {
 
 struct SCtgTask {
   CTG_TASK_TYPE   type;
+  bool            subTask;
   int32_t         taskId;
   SCtgJob*        pJob;
   void*           taskCtx;
@@ -623,6 +637,7 @@ typedef struct SCtgCacheItemInfo {
 #define CTG_FLAG_SYS_DB       0x8
 #define CTG_FLAG_FORCE_UPDATE 0x10
 #define CTG_FLAG_ONLY_CACHE   0x20
+#define CTG_FLAG_SYNC_OP      0x40
 
 #define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v))
 
@@ -925,6 +940,10 @@ void    ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCach
 void    ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache);
 void    ctgGetGlobalCacheStat(SCtgCacheStat* pStat);
 int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res);
+int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
+int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName);
+int32_t ctgGetTbTagCb(SCtgTask* pTask);
+int32_t ctgGetUserCb(SCtgTask* pTask);
 
 extern SCatalogMgmt      gCtgMgmt;
 extern SCtgDebug         gCTGDebug;

source/libs/catalog/src/catalog.c

@@ -208,7 +208,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
   }
 
   while (true) {
-    CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, false));
+    CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, ctx->flag & CTG_FLAG_SYNC_OP));
 
     if (CTG_IS_META_TABLE(output->metaType)) {
       *pTableMeta = output->tbMeta;
@@ -429,6 +429,48 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
   CTG_RET(TSDB_CODE_SUCCESS);
 }
 
+int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) {
+  SVgroupInfo vgroupInfo = {0};
+  STableCfg* pCfg = NULL;
+  int32_t code = 0;
+
+  CTG_ERR_RET(ctgGetTbHashVgroup(pCtg, pConn, pTableName, &vgroupInfo, NULL));
+  CTG_ERR_RET(ctgGetTableCfgFromVnode(pCtg, pConn, pTableName, &vgroupInfo, &pCfg, NULL));
+
+  if (NULL == pCfg->pTags || pCfg->tagsLen <= 0) {
+    ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pCfg->pTags, pCfg->tagsLen);
+    CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
+  }
+
+  SArray* pTagVals = NULL;
+  STag*   pTag = (STag*)pCfg->pTags;
+
+  if (tTagIsJson(pTag)) {
+    pTagVals = taosArrayInit(1, sizeof(STagVal));
+    if (NULL == pTagVals) {
+      CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+
+    char* pJson = parseTagDatatoJson(pTag);
+    STagVal tagVal;
+    tagVal.cid = 0;
+    tagVal.type = TSDB_DATA_TYPE_JSON;
+    tagVal.pData = pJson;
+    tagVal.nData = strlen(pJson);
+    taosArrayPush(pTagVals, &tagVal);
+  } else {
+    CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals));
+  }
+
+  *pRes = pTagVals;
+
+_return:
+
+  tFreeSTableCfgRsp((STableCfgRsp*)pCfg);
+  
+  CTG_RET(code);
+}
+
 int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) {
   STableMeta*   tbMeta = NULL;
   int32_t       code = 0;
@@ -1414,6 +1456,21 @@ _return:
   CTG_API_LEAVE(code);
 }
 
+int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
+  CTG_API_ENTER();
+
+  if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) {
+    CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
+  }
+
+  int32_t code = 0;
+  CTG_ERR_JRET(ctgGetTbTag(pCtg, pConn, (SName*)pTableName, pRes));
+
+_return:
+
+  CTG_API_LEAVE(code);
+}
+
 int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) {
   CTG_API_ENTER();
 

source/libs/catalog/src/ctgAsync.c

@@ -21,7 +21,8 @@
 #include "trpc.h"
 
 int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
-  SName*   name = (SName*)param;
+  SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param;
+  SName*   name = pParam->pName;
   SCtgTask task = {0};
 
   task.type = CTG_TASK_GET_TB_META;
@@ -41,7 +42,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   }
 
   memcpy(ctx->pName, name, sizeof(*name));
-  ctx->flag = CTG_FLAG_UNKNOWN_STB;
+  ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB;
 
   taosArrayPush(pJob->pTasks, &task);
 
@@ -386,6 +387,37 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
+  SName*   name = (SName*)param;
+  SCtgTask task = {0};
+
+  task.type = CTG_TASK_GET_TB_TAG;
+  task.taskId = taskIdx;
+  task.pJob = pJob;
+
+  task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTagCtx));
+  if (NULL == task.taskCtx) {
+    CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+  }
+
+  SCtgTbTagCtx* ctx = task.taskCtx;
+  ctx->pName = taosMemoryMalloc(sizeof(*name));
+  if (NULL == ctx->pName) {
+    taosMemoryFree(task.taskCtx);
+    CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+  }
+
+  memcpy(ctx->pName, name, sizeof(*name));
+
+  taosArrayPush(pJob->pTasks, &task);
+
+  qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx,
+         ctgTaskTypeStr(task.type), name->tname);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) {
   SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@@ -437,6 +469,15 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con
     char   dbFName[TSDB_DB_FNAME_LEN];
     tNameGetFullDbName(name, dbFName);
     taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
+    taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName));
+  }
+
+  for (int32_t i = 0; i < pJob->tbTagNum; ++i) {
+    SName* name = taosArrayGet(pReq->pTableTag, i);
+    char   dbFName[TSDB_DB_FNAME_LEN];
+    tNameGetFullDbName(name, dbFName);
+    taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
+    taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName));
   }
 
   char* dbFName = taosHashIterate(pDb, NULL);
@@ -505,9 +546,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
   int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo);
   int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex);
   int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
+  int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag);
 
   int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
-                    userNum + dbInfoNum + tbIndexNum + tbCfgNum;
+                    userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum;
 
   *job = taosMemoryCalloc(1, sizeof(SCtgJob));
   if (NULL == *job) {
@@ -537,6 +579,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
   pJob->tbIndexNum = tbIndexNum;
   pJob->tbCfgNum = tbCfgNum;
   pJob->svrVerNum = svrVerNum;
+  pJob->tbTagNum = tbTagNum;
 
 #if CTG_BATCH_FETCH
   pJob->pBatchs =
@@ -604,6 +647,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
     CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL));
   }
 
+  for (int32_t i = 0; i < tbTagNum; ++i) {
+    SName* name = taosArrayGet(pReq->pTableTag, i);
+    CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL));
+  }
+
+
   for (int32_t i = 0; i < indexNum; ++i) {
     char* indexName = taosArrayGet(pReq->pIndex, i);
     CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL));
@@ -650,6 +699,10 @@ _return:
 }
 
 int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+  
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableMeta) {
     pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes));
@@ -665,6 +718,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpTbMetasRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
 
   pJob->jobRes.pTableMeta = pTask->res;
@@ -673,6 +730,10 @@ int32_t ctgDumpTbMetasRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbVgroup) {
     pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes));
@@ -688,6 +749,10 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableHash) {
     pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes));
@@ -703,6 +768,10 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpTbHashsRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
 
   pJob->jobRes.pTableHash = pTask->res;
@@ -711,9 +780,17 @@ int32_t ctgDumpTbHashsRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpTbIndexRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableIndex) {
-    pJob->jobRes.pTableIndex = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes));
+    SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes));
+    if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableIndex, NULL, pRes)) {
+      taosArrayDestroy(pRes);
+    }
+
     if (NULL == pJob->jobRes.pTableIndex) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
@@ -726,9 +803,17 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpTbCfgRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableCfg) {
-    pJob->jobRes.pTableCfg = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes));
+    SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes));
+    if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableCfg, NULL, pRes)) {
+      taosArrayDestroy(pRes);
+    }
+
     if (NULL == pJob->jobRes.pTableCfg) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
@@ -740,7 +825,35 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t ctgDumpTbTagRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SCtgJob* pJob = pTask->pJob;
+  if (NULL == pJob->jobRes.pTableTag) {
+    SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes));
+    if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableTag, NULL, pRes)) {
+      taosArrayDestroy(pRes);
+    }
+    
+    if (NULL == pJob->jobRes.pTableTag) {
+      CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
+
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableTag, &res);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t ctgDumpIndexRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pIndex) {
     pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes));
@@ -756,6 +869,10 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pQnodeList) {
     pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes));
@@ -771,6 +888,10 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpDnodeRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDnodeList) {
     pJob->jobRes.pDnodeList = taosArrayInit(1, sizeof(SMetaRes));
@@ -786,6 +907,10 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbCfg) {
     pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes));
@@ -801,6 +926,10 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbInfo) {
     pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes));
@@ -816,6 +945,10 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpUdfRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUdfList) {
     pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes));
@@ -831,6 +964,10 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpUserRes(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUser) {
     pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes));
@@ -846,6 +983,10 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) {
 }
 
 int32_t ctgDumpSvrVer(SCtgTask* pTask) {
+  if (pTask->subTask) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pSvrVer) {
     pJob->jobRes.pSvrVer = taosMemoryCalloc(1, sizeof(SMetaRes));
@@ -1075,7 +1216,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf
 
   STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out;
 
-  ctgUpdateTbMetaToCache(pCtg, pOut, false);
+  ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP);
 
   if (CTG_IS_META_BOTH(pOut->metaType)) {
     memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta));
@@ -1473,6 +1614,49 @@ _return:
   CTG_RET(code);
 }
 
+
+int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
+  int32_t   code = 0;
+  SCtgTask* pTask = tReq->pTask;
+  SCatalog* pCtg = pTask->pJob->pCtg;
+  CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target));
+
+  STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out;
+  if (NULL == pRsp->pTags || pRsp->tagsLen <= 0) {
+    ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pRsp->pTags, pRsp->tagsLen);
+    CTG_ERR_JRET(TSDB_CODE_INVALID_MSG);
+  }
+
+  SArray* pTagVals = NULL;
+  STag*   pTag = (STag*)pRsp->pTags;
+
+  if (tTagIsJson(pTag)) {
+    pTagVals = taosArrayInit(1, sizeof(STagVal));
+    if (NULL == pTagVals) {
+      CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+
+    char* pJson = parseTagDatatoJson(pTag);
+    STagVal tagVal;
+    tagVal.cid = 0;
+    tagVal.type = TSDB_DATA_TYPE_JSON;
+    tagVal.pData = pJson;
+    tagVal.nData = strlen(pJson);
+    taosArrayPush(pTagVals, &tagVal);
+  } else {
+    CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals));
+  }
+  
+  pTask->res = pTagVals;
+
+_return:
+
+  ctgHandleTaskEnd(pTask, code);
+
+  CTG_RET(code);
+}
+
+
 int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
   int32_t   code = 0;
   SCtgTask* pTask = tReq->pTask;
@@ -1905,7 +2089,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) {
   if (pCtx->tbType <= 0) {
     CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType));
     if (pCtx->tbType <= 0) {
-      CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, pCtx->pName));
+      SCtgTbMetaParam param;
+      param.pName = pCtx->pName;
+      param.flag = 0;
+      CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &param));
       return TSDB_CODE_SUCCESS;
     }
   }
@@ -1935,6 +2122,45 @@ _return:
   CTG_RET(code);
 }
 
+
+int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) {
+  int32_t           code = 0;
+  SCatalog*         pCtg = pTask->pJob->pCtg;
+  SRequestConnInfo* pConn = &pTask->pJob->conn;
+  SCtgTbTagCtx*     pCtx = (SCtgTbTagCtx*)pTask->taskCtx;
+  SArray*           pRes = NULL;
+  char              dbFName[TSDB_DB_FNAME_LEN];
+  tNameGetFullDbName(pCtx->pName, dbFName);
+  SCtgJob*    pJob = pTask->pJob;
+  SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1);
+  if (NULL == pMsgCtx->pBatchs) {
+    pMsgCtx->pBatchs = pJob->pBatchs;
+  }
+
+  if (NULL == pCtx->pVgInfo) {
+    CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo));
+    if (NULL == pCtx->pVgInfo) {
+      CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName));
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  CTG_CACHE_NHIT_INC(CTG_CI_TBL_TAG, 1);
+
+  CTG_ERR_JRET(ctgGetTableCfgFromVnode(pCtg, pConn, pCtx->pName, pCtx->pVgInfo, NULL, pTask));
+
+  return TSDB_CODE_SUCCESS;
+
+_return:
+
+  if (CTG_TASK_LAUNCHED == pTask->status) {
+    ctgHandleTaskEnd(pTask, code);
+  }
+
+  CTG_RET(code);
+}
+
+
 int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) {
   SCatalog*         pCtg = pTask->pJob->pCtg;
   SRequestConnInfo* pConn = &pTask->pJob->conn;
@@ -2077,6 +2303,8 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
   if (inCache) {
     pTask->res = rsp.pRawRes;
 
+    ctgTaskDebug("Final res got, pass:%d, pCond:%p", rsp.pRawRes->pass, rsp.pRawRes->pCond);
+
     CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
     return TSDB_CODE_SUCCESS;
   }
@@ -2084,7 +2312,10 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
   taosMemoryFreeClear(rsp.pRawRes);
 
   if (rsp.metaNotExists) {
-    CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName));
+    SCtgTbMetaParam param;
+    param.pName = &pCtx->user.tbName;
+    param.flag = CTG_FLAG_SYNC_OP;
+    CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, &param));
   } else {
     CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask));
   }
@@ -2138,6 +2369,27 @@ _return:
   CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code));
 }
 
+int32_t ctgGetTbTagCb(SCtgTask* pTask) {
+  int32_t code = 0;
+
+  CTG_ERR_JRET(pTask->subRes.code);
+
+  SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx;
+  SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res;
+
+  if (NULL == pCtx->pVgInfo) {
+    pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo));
+    CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo));
+  }
+  
+  CTG_RET(ctgLaunchGetTbTagTask(pTask));
+
+_return:
+
+  CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code));
+}
+
+
 
 int32_t ctgGetUserCb(SCtgTask* pTask) {
   int32_t code = 0;
@@ -2162,8 +2414,12 @@ int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) {
 
 int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) {
   SCtgTbMetaCtx* ctx = pTask->taskCtx;
+  SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param;
 
-  *equal = tNameTbNameEqual(ctx->pName, (SName*)param);
+  *equal = tNameTbNameEqual(ctx->pName, (SName*)pParam->pName);
+  if (*equal) {
+    ctx->flag |= pParam->flag;
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -2197,6 +2453,7 @@ SCtgAsyncFps gCtgAsyncFps[] = {
     {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL},
     {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL},
     {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL},
+    {ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL},
 };
 
 int32_t ctgMakeAsyncRes(SCtgJob* pJob) {
@@ -2284,6 +2541,9 @@ int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp,
   }
 
   SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId);
+  if (newTask) {
+    pSub->subTask = true;
+  }
 
   CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask));
 

source/libs/catalog/src/ctgCache.c

@@ -703,7 +703,31 @@ _return:
   CTG_RET(code);
 }
 
+int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) {
+  *stbName = NULL;
+
+  SCtgDBCache *dbCache = NULL;
+  ctgAcquireDBCache(pCtg, dbFName, &dbCache);
+  if (NULL == dbCache) {
+    ctgDebug("db %s not in cache", dbFName);
+    return TSDB_CODE_SUCCESS;
+  }  
+  
+  char *stb = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid));
+  if (NULL == stb) {
+    ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  *stbName = taosStrdup(stb);
+
+  taosHashRelease(dbCache->stbCache, stb);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) {
+  int32_t code = 0;
   if (IS_SYS_DBNAME(pReq->tbName.dbname)) {
     *inCache = true;
     pRes->pRawRes->pass = true;
@@ -728,7 +752,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache,
 
   CTG_LOCK(CTG_READ, &pUser->lock);
   memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth));
-  int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes);
+  code = ctgChkSetAuthRes(pCtg, &req, pRes);
   CTG_UNLOCK(CTG_READ, &pUser->lock);
   CTG_ERR_JRET(code);
 
@@ -742,8 +766,9 @@ _return:
 
   *inCache = false;
   CTG_CACHE_NHIT_INC(CTG_CI_USER, 1);
+  ctgDebug("Get user from cache failed, user:%s, metaNotExists:%d, code:%d", pReq->user, pRes->metaNotExists, code);
 
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 void ctgDequeue(SCtgCacheOperation **op) {

source/libs/catalog/src/ctgUtil.c

@@ -170,6 +170,9 @@ void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pTableCfg);
   pData->pTableCfg = NULL;
 
+  taosArrayDestroy(pData->pTableTag);
+  pData->pTableTag = NULL;
+
   taosMemoryFreeClear(pData->pSvrVer);
 }
 
@@ -486,6 +489,18 @@ void ctgFreeBatchHash(void* hash) {
   taosMemoryFreeClear(pRes->pRes);
 }
 
+void ctgFreeJsonTagVal(void* val) {
+  if (NULL == val) {
+    return;
+  }
+
+  STagVal* pVal = (STagVal*)val;
+
+  if (TSDB_DATA_TYPE_JSON == pVal->type) {
+    taosMemoryFree(pVal->pData);
+  }
+}
+
 void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
   switch (type) {
     case CTG_TASK_GET_QNODE:
@@ -516,16 +531,32 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
       }
       break;
     }
+    case CTG_TASK_GET_USER: {
+      if (*pRes) {
+        SUserAuthRes* pAuth = (SUserAuthRes*)*pRes;
+        nodesDestroyNode(pAuth->pCond);
+        taosMemoryFreeClear(*pRes);
+      }
+      break;
+    }
     case CTG_TASK_GET_TB_HASH:
     case CTG_TASK_GET_DB_INFO:
     case CTG_TASK_GET_INDEX_INFO:
     case CTG_TASK_GET_UDF:
-    case CTG_TASK_GET_USER:
     case CTG_TASK_GET_SVR_VER:
     case CTG_TASK_GET_TB_META: {
       taosMemoryFreeClear(*pRes);
       break;
     }
+    case CTG_TASK_GET_TB_TAG: {
+      if (1 == taosArrayGetSize(*pRes)) {
+        taosArrayDestroyEx(*pRes, ctgFreeJsonTagVal);
+      } else {
+        taosArrayDestroy(*pRes);
+      }
+      *pRes = NULL;
+      break;
+    }
     case CTG_TASK_GET_TB_META_BATCH: {
       SArray* pArray = (SArray*)*pRes;
       int32_t num = taosArrayGetSize(pArray);
@@ -679,6 +710,13 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
       taosMemoryFreeClear(pTask->taskCtx);
       break;
     }
+    case CTG_TASK_GET_TB_TAG: {
+      SCtgTbTagCtx* taskCtx = (SCtgTbTagCtx*)pTask->taskCtx;
+      taosMemoryFreeClear(taskCtx->pName);
+      taosMemoryFreeClear(taskCtx->pVgInfo);
+      taosMemoryFreeClear(taskCtx);
+      break;
+    }
     case CTG_TASK_GET_DB_VGROUP:
     case CTG_TASK_GET_DB_CFG:
     case CTG_TASK_GET_DB_INFO:
@@ -1336,57 +1374,75 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
   STableMeta*      pMeta = NULL;
   SGetUserAuthRsp* pInfo = &req->authInfo;
   SHashObj*        pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs;
+  char*            stbName = NULL;
 
-  char tbFullName[TSDB_TABLE_FNAME_LEN];
+  char tbFName[TSDB_TABLE_FNAME_LEN];
-  tNameExtractFullName(&req->pRawReq->tbName, tbFullName);
+  char dbFName[TSDB_DB_FNAME_LEN];
-  char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName));
+  tNameExtractFullName(&req->pRawReq->tbName, tbFName);
+  tNameGetFullDbName(&req->pRawReq->tbName, dbFName);
+
+  while (true) {
+    taosMemoryFreeClear(pMeta);
+
+    char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName));
     if (pCond) {
       if (strlen(pCond) > 1) {
-      CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond));
+        CTG_ERR_JRET(nodesStringToNode(pCond, &res->pRawRes->pCond));
       }
 
       res->pRawRes->pass = true;
-    return TSDB_CODE_SUCCESS;
+      goto _return;
     }
 
+    if (stbName) {
       res->pRawRes->pass = false;
+      goto _return;
+    }
 
-  // CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta));
+    CTG_ERR_JRET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta));
-  // if (NULL == pMeta) {
+    if (NULL == pMeta) {
-  //   if (req->onlyCache) {
+      if (req->onlyCache) {
-  //     res->metaNotExists = true;
+        res->metaNotExists = true;
-  //     ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
+        ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
-  //     return TSDB_CODE_SUCCESS;
+        goto _return;
-  //   }
+      }
 
-  //   CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta));
+      SCtgTbMetaCtx ctx = {0};
-  // }
+      ctx.pName = (SName*)&req->pRawReq->tbName;
+      ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP;
 
-  // if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) {
+      CTG_ERR_JRET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta));
-  //   res->pRawRes->pass = false;
+    }
-  //   goto _return;
-  // }
 
-  // if (TSDB_CHILD_TABLE == pMeta->tableType) {
+    if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) {
-  //   res->pRawRes->pass = true;
+      res->pRawRes->pass = false;
+      goto _return;
+    }
 
-  //   /*
+    if (TSDB_CHILD_TABLE == pMeta->tableType) {
-  //       char stbName[TSDB_TABLE_NAME_LEN] = {0};
+      CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, dbFName, pMeta->suid, &stbName));
-  //       CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName));
+      if (NULL == stbName) {
-  //       if (0 == stbName[0]) {
+        if (req->onlyCache) {
-  //         if (req->onlyCache) {
+          res->metaNotExists = true;
-  //           res->notExists = true;
+          ctgDebug("suid %" PRIu64 " name not in cache for auth", pMeta->suid);
-  //           return TSDB_CODE_SUCCESS;
+          goto _return;
-  //         }
+        }
 
-  //         CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0));
+        continue;
-  //       }
+      }
-  //   */
+
-  // }
+      sprintf(tbFName, "%s.%s", dbFName, stbName);
+      continue;
+    }
+
+    ctgError("Invalid table type %d for %s", pMeta->tableType, tbFName);
+    CTG_ERR_JRET(TSDB_CODE_INVALID_PARA);
+  }
 
 _return:
 
   taosMemoryFree(pMeta);
+  taosMemoryFree(stbName);
 
   CTG_RET(code);
 }
@@ -1423,7 +1479,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
       if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) {
         req->singleType = AUTH_TYPE_READ;
         CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
-        if (pRes->pass) {
+        if (pRes->pass || res->metaNotExists) {
           return TSDB_CODE_SUCCESS;
         }
       }
@@ -1439,7 +1495,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
       if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) {
         req->singleType = AUTH_TYPE_WRITE;
         CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
-        if (pRes->pass) {
+        if (pRes->pass || res->metaNotExists) {
           return TSDB_CODE_SUCCESS;
         }
       }

source/libs/command/src/command.c

@@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
 
   char* retentions = buildRetension(pCfg->pRetensions);
   int32_t dbFNameLen = strlen(dbFName);
-  int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0;
+  int32_t hashPrefix = 0;
+  if (pCfg->hashPrefix > 0) {
+    hashPrefix = pCfg->hashPrefix - dbFNameLen - 1;
+  } else if (pCfg->hashPrefix < 0) {
+    hashPrefix = pCfg->hashPrefix + dbFNameLen + 1;
+  }
 
   len += sprintf(
       buf2 + VARSTR_HEADER_SIZE,

source/libs/executor/src/executor.c

@@ -1108,6 +1108,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       pScanBaseInfo->dataReader = NULL;
 
       // let's seek to the next version in wal file
+      int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal);
+      if (pOffset->version + 1 < firstVer){
+        pOffset->version = firstVer - 1;
+      }
+
       if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
         qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
         return -1;

source/libs/executor/src/sysscanoperator.c

@@ -1482,11 +1482,7 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) {
           pInfo->pIdx->init = 1;
           SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator);
           return blk;
-        } else if (flt == -2) {
+        } else if ((flt == -1) || (flt == -2)) {
-          qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo));
-          return NULL;
-        } else if (flt == -1) {
-          // not idx
           qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo));
         }
       } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) {

source/libs/executor/src/timewindowoperator.c

@@ -2541,6 +2541,20 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   }
 
   while (1) {
+    if (isTaskKilled(pTaskInfo)) {
+
+      if (pInfo->pUpdated != NULL) {
+        pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
+      }
+
+      if (pInfo->pUpdatedMap != NULL) {
+        tSimpleHashCleanup(pInfo->pUpdatedMap);
+        pInfo->pUpdatedMap = NULL;
+      }
+
+      T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
+    }
+
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
       pOperator->status = OP_RES_TO_RETURN;
@@ -2635,6 +2649,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {
     taosArrayPush(pInfo->pUpdated, pIte);
   }
+
   tSimpleHashCleanup(pInfo->pUpdatedMap);
   pInfo->pUpdatedMap = NULL;
   taosArraySort(pInfo->pUpdated, winKeyCmprImpl);

source/libs/executor/src/tsort.c

@@ -155,7 +155,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
 
   int64_t fetchUs = 0, fetchNum = 0;
   tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum);
-  qError("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
+  qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
   
   taosArrayDestroy(pSortHandle->pOrderedSource);
   taosMemoryFreeClear(pSortHandle);
@@ -316,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32
     }
 
     int64_t et = taosGetTimestampUs();
-    qError("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
+    qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
   }
 
   return code;

source/libs/function/CMakeLists.txt

@@ -79,6 +79,26 @@ ENDIF ()
 target_link_libraries(
     udf1 PUBLIC os ${LINK_JEMALLOC})
 
+
+add_library(udf1_dup STATIC MODULE test/udf1_dup.c)
+target_include_directories(
+        udf1_dup
+        PUBLIC
+            "${TD_SOURCE_DIR}/include/libs/function"
+            "${TD_SOURCE_DIR}/include/util"
+            "${TD_SOURCE_DIR}/include/common"
+            "${TD_SOURCE_DIR}/include/client"
+            "${TD_SOURCE_DIR}/include/os"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(udf1_dup jemalloc)
+ENDIF ()
+
+target_link_libraries(
+    udf1_dup PUBLIC os ${LINK_JEMALLOC})
+
 add_library(udf2 STATIC MODULE test/udf2.c)
 target_include_directories(
         udf2
@@ -99,6 +119,26 @@ target_link_libraries(
     udf2 PUBLIC os ${LINK_JEMALLOC}
 )
 
+add_library(udf2_dup STATIC MODULE test/udf2_dup.c)
+target_include_directories(
+        udf2_dup
+        PUBLIC
+            "${TD_SOURCE_DIR}/include/libs/function"
+            "${TD_SOURCE_DIR}/include/util"
+            "${TD_SOURCE_DIR}/include/common"
+            "${TD_SOURCE_DIR}/include/client"
+            "${TD_SOURCE_DIR}/include/os"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+    ADD_DEPENDENCIES(udf2_dup jemalloc)
+ENDIF ()
+
+target_link_libraries(
+    udf2_dup PUBLIC os ${LINK_JEMALLOC}
+)
+
 #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin)
 add_executable(udfd src/udfd.c)
 target_include_directories(

source/libs/function/test/udf1_dup.c

@@ -0,0 +1,42 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifdef LINUX
+#include <unistd.h>
+#endif
+#ifdef WINDOWS
+#include <windows.h>
+#endif
+#include "taosudf.h"
+
+
+DLL_EXPORT int32_t udf1_dup_init() { return 0; }
+
+DLL_EXPORT int32_t udf1_dup_destroy() { return 0; }
+
+DLL_EXPORT int32_t udf1_dup(SUdfDataBlock *block, SUdfColumn *resultCol) {
+  SUdfColumnData *resultData = &resultCol->colData;
+  for (int32_t i = 0; i < block->numOfRows; ++i) {
+    int j = 0;
+    for (; j < block->numOfCols; ++j) {
+      if (udfColDataIsNull(block->udfCols[j], i)) {
+        udfColDataSetNull(resultCol, i);
+        break;
+      }
+    }
+    if (j == block->numOfCols) {
+      int32_t luckyNum = 2;
+      udfColDataSet(resultCol, i, (char *)&luckyNum, false);
+    }
+  }
+  // to simulate actual processing delay by udf
+#ifdef LINUX
+  usleep(1 * 1000);  // usleep takes sleep time in us (1 millionth of a second)
+#endif
+#ifdef WINDOWS
+  Sleep(1);
+#endif
+  resultData->numOfRows = block->numOfRows;
+  return 0;
+}
+

source/libs/function/test/udf2_dup.c

@@ -0,0 +1,78 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "taosudf.h"
+
+DLL_EXPORT int32_t udf2_dup_init() { return 0; }
+
+DLL_EXPORT int32_t udf2_dup_destroy() { return 0; }
+
+DLL_EXPORT int32_t udf2_dup_start(SUdfInterBuf* buf) {
+  *(int64_t*)(buf->buf) = 0;
+  buf->bufLen = sizeof(double);
+  buf->numOfResult = 1;
+  return 0;
+}
+
+DLL_EXPORT int32_t udf2_dup(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) {
+  double sumSquares = 0;
+  if (interBuf->numOfResult == 1) {
+    sumSquares = *(double*)interBuf->buf;
+  }
+  int8_t numNotNull = 0;
+  for (int32_t i = 0; i < block->numOfCols; ++i) {
+    SUdfColumn* col = block->udfCols[i];
+    if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) {
+      return TSDB_CODE_UDF_INVALID_INPUT;
+    }
+  }
+  for (int32_t i = 0; i < block->numOfCols; ++i) {
+    for (int32_t j = 0; j < block->numOfRows; ++j) {
+      SUdfColumn* col = block->udfCols[i];
+      if (udfColDataIsNull(col, j)) {
+        continue;
+      }
+      switch (col->colMeta.type) {
+        case TSDB_DATA_TYPE_INT: {
+          char*   cell = udfColDataGetData(col, j);
+          int32_t num = *(int32_t*)cell;
+          sumSquares += (double)num * num;
+          break;
+        }
+        case TSDB_DATA_TYPE_DOUBLE: {
+          char*  cell = udfColDataGetData(col, j);
+          double num = *(double*)cell;
+          sumSquares += num * num;
+          break;
+        }
+        default:
+          break;
+      }
+      ++numNotNull;
+    }
+  }
+
+  *(double*)(newInterBuf->buf) = sumSquares;
+  newInterBuf->bufLen = sizeof(double);
+
+  if (interBuf->numOfResult == 0 && numNotNull == 0) {
+    newInterBuf->numOfResult = 0;
+  } else {
+    newInterBuf->numOfResult = 1;
+  }
+  return 0;
+}
+
+DLL_EXPORT int32_t udf2_dup_finish(SUdfInterBuf* buf, SUdfInterBuf* resultData) {
+  if (buf->numOfResult == 0) {
+    resultData->numOfResult = 0;
+    return 0;
+  }
+  double sumSquares = *(double*)(buf->buf);
+  *(double*)(resultData->buf) = sqrt(sumSquares) + 100;
+  resultData->bufLen = sizeof(double);
+  resultData->numOfResult = 1;
+  return 0;
+}

source/libs/index/src/indexFilter.c

@@ -221,6 +221,71 @@ static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCt
   param->status = SFLT_COARSE_INDEX;
   return 0;
 }
+static int32_t sifNeedConvertCond(SNode *l, SNode *r) {
+  if (nodeType(l) != QUERY_NODE_COLUMN || nodeType(r) != QUERY_NODE_VALUE) {
+    return 0;
+  }
+  SColumnNode *c = (SColumnNode *)l;
+  SValueNode  *v = (SValueNode *)r;
+  int32_t      ctype = c->node.resType.type;
+  int32_t      vtype = v->node.resType.type;
+  if (!IS_VAR_DATA_TYPE(ctype) && IS_VAR_DATA_TYPE(vtype)) {
+    return 1;
+  }
+  return 0;
+}
+static int32_t sifInitParamValByCol(SNode *r, SNode *l, SIFParam *param, SIFCtx *ctx) {
+  param->status = SFLT_COARSE_INDEX;
+  SColumnNode *cn = (SColumnNode *)r;
+  SValueNode  *vn = (SValueNode *)l;
+  if (vn->typeData == TSDB_DATA_TYPE_NULL && (vn->literal == NULL || strlen(vn->literal) == 0)) {
+    param->status = SFLT_NOT_INDEX;
+    return 0;
+  }
+  SDataType *pType = &cn->node.resType;
+  int32_t    type = pType->type;
+
+  SDataType *pVType = &vn->node.resType;
+  int32_t    vtype = pVType->type;
+  char      *pData = nodesGetValueFromNode(vn);
+  int32_t    valLen = 0;
+  char     **value = &param->condValue;
+
+  if (IS_VAR_DATA_TYPE(type)) {
+    int32_t dataLen = varDataTLen(pData);
+    if (type == TSDB_DATA_TYPE_JSON) {
+      if (*pData == TSDB_DATA_TYPE_NULL) {
+        dataLen = 0;
+      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData);
+      } else if (*pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = LONG_BYTES;
+      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES;
+      }
+      dataLen += CHAR_BYTES;
+    }
+    valLen = dataLen;
+  } else {
+    valLen = pType->bytes;
+  }
+  char *tv = taosMemoryCalloc(1, valLen + 1);
+  if (tv == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  memcpy(tv, pData, valLen);
+  *value = tv;
+
+  param->colId = -1;
+  param->colValType = (uint8_t)(vn->node.resType.type);
+  if (vn->literal != NULL && strlen(vn->literal) <= sizeof(param->colName)) {
+    memcpy(param->colName, vn->literal, strlen(vn->literal));
+  } else {
+    param->status = SFLT_NOT_INDEX;
+  }
+  return 0;
+}
 static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   param->status = SFLT_COARSE_INDEX;
   switch (nodeType(node)) {
@@ -317,8 +382,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
     return TSDB_CODE_SUCCESS;
   } else {
     SIF_ERR_JRET(sifInitParam(node->pLeft, &paramList[0], ctx));
+
     if (nParam > 1) {
+      // if (sifNeedConvertCond(node->pLeft, node->pRight)) {
+      // SIF_ERR_JRET(sifInitParamValByCol(node->pLeft, node->pRight, &paramList[1], ctx));
+      // } else {
       SIF_ERR_JRET(sifInitParam(node->pRight, &paramList[1], ctx));
+      // }
       // if (paramList[0].colValType == TSDB_DATA_TYPE_JSON &&
       //    ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) {
       //  return TSDB_CODE_OUT_OF_MEMORY;
@@ -404,60 +474,149 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve
   }
   return NULL;
 }
+int32_t sifStr2Num(char *buf, int32_t len, int8_t type, void *val) {
+  // signed/unsigned/float
+  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    int64_t v = 0;
+    if (0 != toInteger(buf, len, 10, &v)) {
+      return -1;
+    }
+    if (type == TSDB_DATA_TYPE_BIGINT) {
+      *(int64_t *)val = v;
+    } else if (type == TSDB_DATA_TYPE_INT) {
+      *(int32_t *)val = v;
+    } else if (type == TSDB_DATA_TYPE_TINYINT) {
+      *(int8_t *)val = v;
+    } else if (type == TSDB_DATA_TYPE_SMALLINT) {
+      *(int16_t *)val = v;
+    }
+  } else if (IS_FLOAT_TYPE(type)) {
+    if (type == TSDB_DATA_TYPE_FLOAT) {
+      *(float *)val = taosStr2Float(buf, NULL);
+    } else {
+      *(double *)val = taosStr2Double(buf, NULL);
+    }
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+    uint64_t v = 0;
+    if (0 != toUInteger(buf, len, 10, &v)) {
+      return -1;
+    }
+    if (type == TSDB_DATA_TYPE_UBIGINT) {
+      *(uint64_t *)val = v;
+    } else if (type == TSDB_DATA_TYPE_UINT) {
+      *(uint32_t *)val = v;
+    } else if (type == TSDB_DATA_TYPE_UTINYINT) {
+      *(uint8_t *)val = v;
+    } else if (type == TSDB_DATA_TYPE_USMALLINT) {
+      *(uint16_t *)val = v;
+    }
+  } else {
+    return -1;
+  }
+  return 0;
+}
 
-static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) {
+static int32_t sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) {
+  int32_t code = 0;
   int8_t  ltype = left->colValType, rtype = right->colValType;
+  if (!IS_NUMERIC_TYPE(ltype) || !((IS_NUMERIC_TYPE(rtype)) || rtype == TSDB_DATA_TYPE_VARCHAR)) {
+    return -1;
+  }
   if (ltype == TSDB_DATA_TYPE_FLOAT) {
     float f = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, f);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_FLOAT, &f));
+    }
     typedata->f = f;
     param->val = &typedata->f;
   } else if (ltype == TSDB_DATA_TYPE_DOUBLE) {
     double d = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, d);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_DOUBLE, &d));
+    }
     typedata->d = d;
     param->val = &typedata->d;
   } else if (ltype == TSDB_DATA_TYPE_BIGINT) {
     int64_t i64 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, i64);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_BIGINT, &i64));
+    }
     typedata->i64 = i64;
     param->val = &typedata->i64;
   } else if (ltype == TSDB_DATA_TYPE_INT) {
     int32_t i32 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, i32);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_INT, &i32));
+    }
     typedata->i32 = i32;
     param->val = &typedata->i32;
   } else if (ltype == TSDB_DATA_TYPE_SMALLINT) {
     int16_t i16 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, i16);
+    } else {
+      SIF_ERR_RET(
+          sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_SMALLINT, &i16));
+    }
+
     typedata->i16 = i16;
     param->val = &typedata->i16;
   } else if (ltype == TSDB_DATA_TYPE_TINYINT) {
     int8_t i8 = 0;
-    SIF_DATA_CONVERT(rtype, right->condValue, i8)
+    if (IS_NUMERIC_TYPE(rtype)) {
+      SIF_DATA_CONVERT(rtype, right->condValue, i8);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_TINYINT, &i8));
+    }
     typedata->i8 = i8;
     param->val = &typedata->i8;
   } else if (ltype == TSDB_DATA_TYPE_UBIGINT) {
     uint64_t u64 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, u64);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UBIGINT, &u64));
+    }
     typedata->u64 = u64;
     param->val = &typedata->u64;
   } else if (ltype == TSDB_DATA_TYPE_UINT) {
     uint32_t u32 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, u32);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UINT, &u32));
+    }
     typedata->u32 = u32;
     param->val = &typedata->u32;
   } else if (ltype == TSDB_DATA_TYPE_USMALLINT) {
     uint16_t u16 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, u16);
+    } else {
+      SIF_ERR_RET(
+          sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_USMALLINT, &u16));
+    }
     typedata->u16 = u16;
     param->val = &typedata->u16;
   } else if (ltype == TSDB_DATA_TYPE_UTINYINT) {
     uint8_t u8 = 0;
+    if (IS_NUMERIC_TYPE(rtype)) {
       SIF_DATA_CONVERT(rtype, right->condValue, u8);
+    } else {
+      SIF_ERR_RET(sifStr2Num(varDataVal(right->condValue), varDataLen(right->condValue), TSDB_DATA_TYPE_UTINYINT, &u8));
+    }
     typedata->u8 = u8;
     param->val = &typedata->u8;
   }
+  return 0;
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
   int             ret = 0;
@@ -498,7 +657,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
         param.val = buf;
       }
     } else {
-      sifSetFltParam(left, right, &typedata, &param);
+      if (sifSetFltParam(left, right, &typedata, &param) != 0) return -1;
     }
     ret = metaFilterTableIds(arg->metaEx, &param, output->result);
   }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -827,6 +827,8 @@ void nodesDestroyNode(SNode* pNode) {
       SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode;
       destroyVgDataBlockArray(pStmt->pDataBlocks);
       taosMemoryFreeClear(pStmt->pTableMeta);
+      nodesDestroyNode(pStmt->pTagCond);
+      taosArrayDestroy(pStmt->pTableTag);
       taosHashCleanup(pStmt->pVgroupsHashObj);
       taosHashCleanup(pStmt->pSubTableHashObj);
       taosHashCleanup(pStmt->pTableNameHashObj);
@@ -953,8 +955,12 @@ void nodesDestroyNode(SNode* pNode) {
       break;
     case QUERY_NODE_SPLIT_VGROUP_STMT:  // no pointer field
     case QUERY_NODE_SYNCDB_STMT:        // no pointer field
-    case QUERY_NODE_GRANT_STMT:         // no pointer field
+      break;
-    case QUERY_NODE_REVOKE_STMT:        // no pointer field
+    case QUERY_NODE_GRANT_STMT:
+      nodesDestroyNode(((SGrantStmt*)pNode)->pTagCond);
+      break;
+    case QUERY_NODE_REVOKE_STMT:
+      nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond);
       break;
     case QUERY_NODE_SHOW_DNODES_STMT:
     case QUERY_NODE_SHOW_MNODES_STMT:

source/libs/parser/inc/sql.y

@@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C).
 db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C).                    { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); }
 db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).                   { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); }
 db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C).                        { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
-db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C).                       { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); }
+db_options(A) ::= db_options(B) TABLE_PREFIX signed(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
-db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C).                       { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); }
+db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
 
 alter_db_options(A) ::= alter_db_option(B).                                       { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
 alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).                   { A = setAlterDatabaseOption(pCxt, B, &C); }

source/libs/parser/src/parAstCreater.c

@@ -1024,12 +1024,28 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
     case DB_OPTION_STT_TRIGGER:
       pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
-    case DB_OPTION_TABLE_PREFIX:
+    case DB_OPTION_TABLE_PREFIX: {
-      pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      SValueNode *pNode = (SValueNode *)pVal;
+      if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
+        pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10);
+      } else {
+        snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type");
+        pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;        
+      }
+      nodesDestroyNode((SNode*)pNode);
       break;
-    case DB_OPTION_TABLE_SUFFIX:
+    }
-      pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+    case DB_OPTION_TABLE_SUFFIX:{
+      SValueNode *pNode = (SValueNode *)pVal;
+      if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
+        pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10);
+      } else {
+        snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type");
+        pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;        
+      }
+      nodesDestroyNode((SNode*)pNode);
       break;
+    }
     default:
       break;
   }

source/libs/parser/src/parAuthenticator.c

@@ -70,7 +70,7 @@ static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) {
   return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
 }
 
-static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) {
+static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) {
   SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
   if (NULL == pLogicCond) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -78,7 +78,7 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) {
   pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
   pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
   pLogicCond->condType = LOGIC_COND_TYPE_AND;
-  int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond);
+  int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, pTagCond);
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere);
   }
@@ -106,7 +106,7 @@ static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) {
     return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy);
   }
 
-  return mergeStableTagCond(pWhere, &pTagCondCopy);
+  return mergeStableTagCond(pWhere, pTagCondCopy);
 }
 
 static EDealRes authSelectImpl(SNode* pNode, void* pContext) {

source/libs/parser/src/parInsertSql.c

@@ -53,6 +53,7 @@ typedef struct SInsertParseContext {
   bool           missCache;
   bool           usingDuplicateTable;
   bool           forceUpdate;
+  bool           needTableTagVal;
 } SInsertParseContext;
 
 typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param);
@@ -577,28 +578,39 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
   if (NULL == pValue) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  pValue->node.resType.type = pVal->type;
+
+  pValue->node.resType = ((SColumnNode*)*pNode)->node.resType;
+  nodesDestroyNode(*pNode);
+  *pNode = (SNode*)pValue;
+
   switch (pVal->type) {
     case TSDB_DATA_TYPE_BOOL:
       pValue->datum.b = *(int8_t*)(&pVal->i64);
+      *(bool*)&pValue->typeData = pValue->datum.b;
       break;
     case TSDB_DATA_TYPE_TINYINT:
       pValue->datum.i = *(int8_t*)(&pVal->i64);
+      *(int8_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_SMALLINT:
       pValue->datum.i = *(int16_t*)(&pVal->i64);
+      *(int16_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_INT:
       pValue->datum.i = *(int32_t*)(&pVal->i64);
+      *(int32_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_BIGINT:
       pValue->datum.i = pVal->i64;
+      pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_FLOAT:
       pValue->datum.d = *(float*)(&pVal->i64);
+      *(float*)&pValue->typeData = pValue->datum.d;
       break;
     case TSDB_DATA_TYPE_DOUBLE:
       pValue->datum.d = *(double*)(&pVal->i64);
+      *(double*)&pValue->typeData = pValue->datum.d;
       break;
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_NCHAR:
@@ -611,18 +623,23 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) {
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:
       pValue->datum.i = pVal->i64;
+      pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_UTINYINT:
       pValue->datum.i = *(uint8_t*)(&pVal->i64);
+      *(uint8_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_USMALLINT:
       pValue->datum.i = *(uint16_t*)(&pVal->i64);
+      *(uint16_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_UINT:
       pValue->datum.i = *(uint32_t*)(&pVal->i64);
+      *(uint32_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_UBIGINT:
       pValue->datum.i = *(uint64_t*)(&pVal->i64);
+      *(uint64_t*)&pValue->typeData = pValue->datum.i;
       break;
     case TSDB_DATA_TYPE_JSON:
     case TSDB_DATA_TYPE_VARBINARY:
@@ -667,16 +684,15 @@ static int32_t checkTagCondResult(SNode* pResult) {
                                                                                     : TSDB_CODE_PAR_PERMISSION_DENIED;
 }
 
-int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) {
+static int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode** pCond) {
-  int32_t code = setTagVal(pTagVals, pTagName, pCond);
+  int32_t code = setTagVal(pTagVals, pTagName, *pCond);
-  SNode*  pNew = NULL;
   if (TSDB_CODE_SUCCESS == code) {
-    code = scalarCalculateConstants(pCond, &pNew);
+    code = scalarCalculateConstants(*pCond, pCond);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = checkTagCondResult(pNew);
+    code = checkTagCondResult(*pCond);
   }
-  nodesDestroyNode(pNew);
+  NODES_DESTORY_NODE(*pCond);
   return code;
 }
 
@@ -716,6 +732,10 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt
     }
   }
 
+  if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pTagCond) {
+    code = checkSubtablePrivilege(pTagVals, pTagName, &pStmt->pTagCond);
+  }
+
   if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) {
     code = tTagNew(pTagVals, 1, false, &pTag);
   }
@@ -843,7 +863,7 @@ static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo*
   pInfo->type = AUTH_TYPE_WRITE;
 }
 
-static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) {
+static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) {
   int32_t       code = TSDB_CODE_SUCCESS;
   SUserAuthInfo authInfo = {0};
   setUserAuthInfo(pCxt, pTbName, &authInfo);
@@ -863,11 +883,28 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache)
       *pMissCache = true;
     } else if (!authRes.pass) {
       code = TSDB_CODE_PAR_PERMISSION_DENIED;
+    } else if (NULL != authRes.pCond) {
+      *pTagCond = authRes.pCond;
     }
   }
   return code;
 }
 
+static int32_t checkAuthForTable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, bool* pNeedTableTagVal) {
+  SNode*  pTagCond = NULL;
+  int32_t code = checkAuth(pCxt, pTbName, pMissCache, &pTagCond);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pNeedTableTagVal = ((*pMissCache) || (NULL != pTagCond));
+    *pMissCache = (NULL != pTagCond);
+  }
+  nodesDestroyNode(pTagCond);
+  return code;
+}
+
+static int32_t checkAuthForStable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) {
+  return checkAuth(pCxt, pTbName, pMissCache, pTagCond);
+}
+
 static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta,
                             bool* pMissCache) {
   SParseContext* pComCxt = pCxt->pComCxt;
@@ -970,7 +1007,7 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache);
+  int32_t code = checkAuthForTable(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pCxt->needTableTagVal);
   if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
     code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache);
   }
@@ -993,7 +1030,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache);
+  int32_t code = checkAuthForStable(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond);
   if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
     code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache);
   }
@@ -1606,6 +1643,8 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod
 static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
   insDestroyBoundColInfo(&pCxt->tags);
   taosMemoryFreeClear(pStmt->pTableMeta);
+  nodesDestroyNode(pStmt->pTagCond);
+  taosArrayDestroy(pStmt->pTableTag);
   tdDestroySVCreateTbReq(pStmt->pCreateTblReq);
   taosMemoryFreeClear(pStmt->pCreateTblReq);
   pCxt->missCache = false;
@@ -1780,14 +1819,18 @@ static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) {
   return code;
 }
 
-static int32_t checkAuthFromMetaData(const SArray* pUsers) {
+static int32_t checkAuthFromMetaData(const SArray* pUsers, SNode** pTagCond) {
   if (1 != taosArrayGetSize(pUsers)) {
     return TSDB_CODE_FAILED;
   }
 
   SMetaRes* pRes = taosArrayGet(pUsers, 0);
   if (TSDB_CODE_SUCCESS == pRes->code) {
-    return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED;
+    SUserAuthRes* pAuth = pRes->pRes;
+    if (NULL != pAuth->pCond) {
+      *pTagCond = nodesCloneNode(pAuth->pCond);
+    }
+    return pAuth->pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED;
   }
   return pRes->code;
 }
@@ -1826,9 +1869,40 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpS
                      sizeof(SVgroupInfo));
 }
 
+static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) {
+  *pTagName = taosArrayInit(pMeta->tableInfo.numOfTags, TSDB_COL_NAME_LEN);
+  if (NULL == *pTagName) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  SSchema* pSchema = getTableTagSchema(pMeta);
+  for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) {
+    taosArrayPush(*pTagName, pSchema[i].name);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t checkSubtablePrivilegeForTable(const SArray* pTables, SVnodeModifyOpStmt* pStmt) {
+  if (1 != taosArrayGetSize(pTables)) {
+    return TSDB_CODE_FAILED;
+  }
+
+  SMetaRes* pRes = taosArrayGet(pTables, 0);
+  if (TSDB_CODE_SUCCESS != pRes->code) {
+    return pRes->code;
+  }
+
+  SArray* pTagName = NULL;
+  int32_t code = buildTagNameFromMeta(pStmt->pTableMeta, &pTagName);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = checkSubtablePrivilege((SArray*)pRes->pRes, pTagName, &pStmt->pTagCond);
+  }
+  taosArrayDestroy(pTagName);
+  return code;
+}
+
 static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData,
                                           SVnodeModifyOpStmt* pStmt, bool isStb) {
-  int32_t code = checkAuthFromMetaData(pMetaData->pUser);
+  int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond);
   if (TSDB_CODE_SUCCESS == code) {
     code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta);
   }
@@ -1841,6 +1915,9 @@ static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMeta
   if (TSDB_CODE_SUCCESS == code) {
     code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb);
   }
+  if (TSDB_CODE_SUCCESS == code && !isStb && NULL != pStmt->pTagCond) {
+    code = checkSubtablePrivilegeForTable(pMetaData->pTableTag, pStmt);
+  }
   return code;
 }
 
@@ -1860,6 +1937,8 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) {
   pCatalogReq->pTableHash = NULL;
   taosArrayDestroy(pCatalogReq->pUser);
   pCatalogReq->pUser = NULL;
+  taosArrayDestroy(pCatalogReq->pTableTag);
+  pCatalogReq->pTableTag = NULL;
 }
 
 static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
@@ -2033,8 +2112,15 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray**
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t buildInsertTableTagReq(SName* pName, SArray** pTables) { return buildInsertTableReq(pName, pTables); }
+
 static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) {
-  int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser);
+  int32_t code = buildInsertUserAuthReq(
+      pCxt->pComCxt->pUser, (0 == pStmt->usingTableName.type ? &pStmt->targetTableName : &pStmt->usingTableName),
+      &pCatalogReq->pUser);
+  if (TSDB_CODE_SUCCESS == code && pCxt->needTableTagVal) {
+    code = buildInsertTableTagReq(&pStmt->targetTableName, &pCatalogReq->pTableTag);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     if (0 == pStmt->usingTableName.type) {
       code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta);

source/libs/parser/src/parTranslater.c

@@ -1310,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
 }
 
 static EDealRes haveVectorFunction(SNode* pNode, void* pContext) {
-  if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || isInterpPseudoColumnFunc(pNode)) {
+  if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) ||
+      isInterpPseudoColumnFunc(pNode)) {
     *((bool*)pContext) = true;
     return DEAL_RES_END;
   }
@@ -4177,6 +4178,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
   return TSDB_CODE_SUCCESS;
 }
 
+
+static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) {
+  if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix,
+                                   TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
+  }
+
+  if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix,
+                                   TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
+  }
+
+  if ((tbPrefix * tbSuffix) < 0) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_prefix & table_suffix: mixed usage not allowed");
+  }
+
+  if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_prefix & table_suffix: exceed max table name length");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
   int32_t daysPerFile = pOptions->daysPerFile;
   int64_t daysToKeep0 = pOptions->keep[0];
@@ -4284,10 +4313,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
     code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
+    code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = checkOptionsDependency(pCxt, pDbName, pOptions);
@@ -6617,6 +6643,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
   if (TSDB_CODE_SUCCESS == code) {
     code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
   }
+  tFreeSAlterUserReq(&req);
   return code;
 }
 

source/libs/planner/src/planOptimizer.c

@@ -2526,7 +2526,7 @@ static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) {
       return false;
     }
   }
-  return true;
+  return LIST_LENGTH(pAggFuncs) > 0;
 }
 
 static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) {

source/libs/qcom/src/querymsg.c

@@ -407,7 +407,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta *
     pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
   }
 
-  qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64
+  qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d"
          " tagNum %d colNum %d precision %d rowSize %d",
          msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName,
          pTableMeta->suid, pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags,

source/libs/scalar/src/scalar.c

@@ -53,6 +53,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara
   int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    colDataDestroy(pColumnData);
     taosMemoryFree(pColumnData);
     return terrno;
   }
@@ -1061,17 +1062,20 @@ int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) {
 
   if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) {
     if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) {
-      SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft));
+      SCL_ERR_JRET(
+          sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode *)node->pLeft));
     }
   } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) {
     if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) {
       if (SCL_IS_VAR_VALUE_NODE(node->pRight)) {
-        SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight));
+        SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight),
+                                             (SValueNode *)node->pRight));
       } else if (QUERY_NODE_NODE_LIST == node->pRight->type) {
         SNode *pNode;
         FOREACH(pNode, ((SNodeListNode *)node->pRight)->pNodeList) {
           if (SCL_IS_VAR_VALUE_NODE(pNode)) {
-            SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode));
+            SCL_ERR_JRET(
+                sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode *)pNode));
           }
         }
       }
@@ -1086,8 +1090,6 @@ _return:
   return DEAL_RES_ERROR;
 }
 
-
-
 int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
   int32_t code = 0;
 
@@ -1108,7 +1110,8 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) {
     SNode *pNode;
     FOREACH(pNode, node->pWhenThenList) {
       if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) {
-        SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, (SValueNode*)((SWhenThenNode *)pNode)->pWhen));
+        SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision,
+                                             (SValueNode *)((SWhenThenNode *)pNode)->pWhen));
       }
     }
   }
@@ -1271,7 +1274,6 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) {
   return DEAL_RES_CONTINUE;
 }
 
-
 EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) {
   SOperatorNode *node = (SOperatorNode *)*pNode;
 

source/libs/stream/src/stream.c

@@ -16,7 +16,7 @@
 #include "streamInc.h"
 #include "ttimer.h"
 
-#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 100000
+#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 3000
 
 int32_t streamInit() {
   int8_t old;
@@ -52,7 +52,7 @@ void streamCleanUp() {
 void streamSchedByTimer(void* param, void* tmrId) {
   SStreamTask* pTask = (void*)param;
 
-  if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
+  if (streamTaskShouldStop(&pTask->status)) {
     streamMetaReleaseTask(NULL, pTask);
     return;
   }

source/libs/stream/src/streamExec.c

@@ -17,6 +17,11 @@
 
 #define STREAM_EXEC_MAX_BATCH_NUM 100
 
+bool streamTaskShouldStop(const SStreamStatus* pStatus) {
+  int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus);
+  return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING);
+}
+
 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) {
   int32_t code = TSDB_CODE_SUCCESS;
   void*   pExecutor = pTask->exec.pExecutor;
@@ -66,7 +71,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
 
   // pExecutor
   while (1) {
-    if (pTask->status.taskStatus == TASK_STATUS__DROPPING) {
+    if (streamTaskShouldStop(&pTask->status)) {
       return 0;
     }
 
@@ -106,7 +111,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
       continue;
     }
 
-    qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId);
+    qDebug("s-task:%s (child %d) executed and get block", pTask->id.idStr, pTask->selfChildId);
 
     SSDataBlock block = {0};
     assignOneDataBlock(&block, output);
@@ -134,7 +139,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
 
     int32_t batchCnt = 0;
     while (1) {
-      if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) {
+      if (streamTaskShouldStop(&pTask->status)) {
         taosArrayDestroy(pRes);
         return 0;
       }
@@ -270,7 +275,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
       }
     }
 
-    if (pTask->status.taskStatus == TASK_STATUS__DROPPING) {
+    if (streamTaskShouldStop(&pTask->status)) {
       if (pInput) {
         streamFreeQitem(pInput);
       }
@@ -301,7 +306,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
              ", checkPoint id:%" PRId64 " -> %" PRId64,
              pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId);
 
-      pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId};
+      pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer};
 
       taosWLockLatch(&pTask->pMeta->lock);
       streamMetaSaveTask(pTask->pMeta, pTask);
@@ -368,7 +373,7 @@ int32_t streamTryExec(SStreamTask* pTask) {
     atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE);
     qDebug("s-task:%s exec completed", pTask->id.idStr);
 
-    if (!taosQueueEmpty(pTask->inputQueue->queue)) {
+    if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status))) {
       streamSchedExec(pTask);
     }
   }

source/libs/stream/src/streamMeta.c

@@ -84,11 +84,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
   tdbClose(pMeta->db);
 
   void* pIter = NULL;
-//  while(pMeta->walScan) {
-//    qDebug("wait stream daemon quit");
-//    taosMsleep(100);
-//  }
-
   while (1) {
     pIter = taosHashIterate(pMeta->pTasks, pIter);
     if (pIter == NULL) {
@@ -102,7 +97,6 @@ void streamMetaClose(SStreamMeta* pMeta) {
     }
 
     tFreeStreamTask(pTask);
-    /*streamMetaReleaseTask(pMeta, pTask);*/
   }
 
   taosHashCleanup(pMeta->pTasks);
@@ -197,11 +191,13 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
   taosRLockLatch(&pMeta->lock);
 
   SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
-  if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) {
+  if (ppTask != NULL) {
+    if (!streamTaskShouldStop(&(*ppTask)->status)) {
       atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
       taosRUnLockLatch(&pMeta->lock);
       return *ppTask;
     }
+  }
 
   taosRUnLockLatch(&pMeta->lock);
   return NULL;
@@ -211,7 +207,7 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1);
   ASSERT(left >= 0);
   if (left == 0) {
-    ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING);
+    ASSERT(streamTaskShouldStop(&pTask->status));
     tFreeStreamTask(pTask);
   }
 }
@@ -222,11 +218,8 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
     SStreamTask* pTask = *ppTask;
     taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
     tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn);
-    /*if (pTask->timer) {
+
-     * taosTmrStop(pTask->timer);*/
+    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP);
-    /*pTask->timer = NULL;*/
-    /*}*/
-    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING);
 
     taosWLockLatch(&pMeta->lock);
     streamMetaReleaseTask(pMeta, pTask);

source/os/src/osFile.c

@@ -538,10 +538,11 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
     return -1;
   }
 
-  struct stat fileStat;
 #ifdef WINDOWS
-  int32_t code = _fstat(pFile->fd, &fileStat);
+  struct __stat64 fileStat;
+  int32_t code = _fstat64(pFile->fd, &fileStat);
 #else
+  struct stat fileStat;
   int32_t code = fstat(pFile->fd, &fileStat);
 #endif
   if (code < 0) {

source/os/src/osLocale.c

@@ -171,7 +171,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) {
     strcpy(outLocale, "en_US.UTF-8");
   } else {
     tstrncpy(outLocale, locale, TD_LOCALE_LEN);
-    // printf("locale not configured, set to system default:%s", outLocale);
+    printf("locale not configured, set to system default:%s\n", outLocale);
   }
 
   // if user does not specify the charset, extract it from locale

source/util/src/tlog.c

@@ -347,7 +347,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
 
   char    name[LOG_FILE_NAME_LEN + 50] = "\0";
   int32_t logstat0_mtime, logstat1_mtime;
-  int32_t size;
 
   tsLogObj.maxLines = maxLines;
   tsLogObj.fileNum = maxFileNum;
@@ -395,8 +394,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
     printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno));
     return -1;
   }
-  size = (int32_t)filesize;
+  tsLogObj.lines = (int32_t)(filesize / 60);
-  tsLogObj.lines = size / 60;
 
   taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END);
 

source/util/src/tsimplehash.c

@@ -361,10 +361,6 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke
   return TSDB_CODE_SUCCESS;
 }
 
-static void destroyItems(void* pItem) {
-  taosMemoryFree(*(void**)pItem);
-}
-
 void tSimpleHashClear(SSHashObj *pHashObj) {
   if (!pHashObj || taosHashTableEmpty(pHashObj)) {
     return;

tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json

@@ -11,6 +11,7 @@
   "confirm_parameter_prompt": "no",
   "prepared_rand": 100,
   "chinese": "no",
+  "escape_character": "yes",
   "insert_interval": 0,
   "num_of_records_per_req": 10,
   "databases": [{
@@ -29,7 +30,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",
@@ -54,7 +54,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb3-2_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",

tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json

@@ -11,6 +11,7 @@
   "confirm_parameter_prompt": "no",
   "prepared_rand": 100,
   "chinese": "no",
+  "escape_character": "yes",
   "insert_interval": 0,
   "num_of_records_per_req": 10,
   "databases": [{
@@ -29,7 +30,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",
@@ -54,7 +54,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb4-2_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",

tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json

@@ -11,6 +11,7 @@
   "confirm_parameter_prompt": "no",
   "prepared_rand": 100,
   "chinese": "no",
+  "escape_character": "yes",
   "insert_interval": 0,
   "num_of_records_per_req": 10,
   "databases": [{
@@ -29,7 +30,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",
@@ -54,7 +54,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb2-2_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",

tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json

@@ -11,6 +11,7 @@
   "confirm_parameter_prompt": "no",
   "prepared_rand": 100,
   "chinese": "no",
+  "escape_character": "yes",
   "insert_interval": 0,
   "num_of_records_per_req": 10,
   "databases": [{
@@ -29,7 +30,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",
@@ -55,7 +55,6 @@
       "child_table_exists":"no",
       "childtable_count": 8,
       "childtable_prefix": "stb1-2_",
-      "escape_character": "yes",
       "auto_create_table": "yes",
       "batch_create_tbl_num": 10,
       "data_source": "rand",

tests/parallel_test/cases.task

@@ -51,6 +51,16 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4
 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
@@ -722,6 +732,7 @@
 ,,y,script,./test.sh -f tsim/user/privilege_db.sim
 ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim
 ,,y,script,./test.sh -f tsim/user/privilege_topic.sim
+,,y,script,./test.sh -f tsim/user/privilege_table.sim
 ,,y,script,./test.sh -f tsim/db/alter_option.sim
 ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim
 ,,y,script,./test.sh -f tsim/db/basic1.sim
@@ -746,6 +757,7 @@
 ,,y,script,./test.sh -f tsim/db/show_create_table.sim
 ,,y,script,./test.sh -f tsim/db/tables.sim
 ,,y,script,./test.sh -f tsim/db/taosdlog.sim
+,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim
 ,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim
 ,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim
 ,,y,script,./test.sh -f tsim/dnode/balance1.sim

tests/parallel_test/run.sh

@@ -303,7 +303,7 @@ function run_thread() {
             if [ ! -z "$corefile" ]; then
                 echo -e "\e[34m corefiles: $corefile \e[0m"
                 local build_dir=$log_dir/build_${hosts[index]}
-                local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
+                local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build"
                 # if [ $ent -ne 0 ]; then
                 #     remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
                 # fi

tests/script/tsim/db/table_prefix_suffix.sim

@@ -0,0 +1,182 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql drop database if exists db1;
+sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2;
+sql use db1;
+sql create table atb1aa (ts timestamp, f1 int);
+sql create table btb1bb (ts timestamp, f1 int);
+sql create table ctb1cc (ts timestamp, f1 int);
+sql create table dtb1dd (ts timestamp, f1 int);
+sql create table atb2aa (ts timestamp, f1 int);
+sql create table btb2bb (ts timestamp, f1 int);
+sql create table ctb2cc (ts timestamp, f1 int);
+sql create table dtb2dd (ts timestamp, f1 int);
+sql create table etb2ee (ts timestamp, f1 int);
+sql show create database db1;
+sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db1;
+ 
+sql drop database if exists db2;
+sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2;
+sql use db2;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table tbbb11 (ts timestamp, f1 int);
+sql create table tccc11 (ts timestamp, f1 int);
+sql create table tddd11 (ts timestamp, f1 int);
+sql create table taaa22 (ts timestamp, f1 int);
+sql create table tbbb22 (ts timestamp, f1 int);
+sql create table tccc22 (ts timestamp, f1 int);
+sql create table tddd22 (ts timestamp, f1 int);
+sql create table teee22 (ts timestamp, f1 int);
+sql show create database db2;
+sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db2;
+ 
+sql drop database if exists db3;
+sql create database db3 vgroups 5 TABLE_PREFIX -1;
+sql use db3;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table tbbb11 (ts timestamp, f1 int);
+sql create table tccc11 (ts timestamp, f1 int);
+sql create table tddd11 (ts timestamp, f1 int);
+sql create table zaaa22 (ts timestamp, f1 int);
+sql create table zbbb22 (ts timestamp, f1 int);
+sql create table zccc22 (ts timestamp, f1 int);
+sql create table zddd22 (ts timestamp, f1 int);
+sql create table zeee22 (ts timestamp, f1 int);
+sql show create database db3;
+sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db3;
+ 
+sql drop database if exists db4;
+sql create database db4 vgroups 5 TABLE_SUFFIX -2;
+sql use db4;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table tbbb11 (ts timestamp, f1 int);
+sql create table tccc11 (ts timestamp, f1 int);
+sql create table tddd11 (ts timestamp, f1 int);
+sql create table zaaa22 (ts timestamp, f1 int);
+sql create table zbbb22 (ts timestamp, f1 int);
+sql create table zccc22 (ts timestamp, f1 int);
+sql create table zddd22 (ts timestamp, f1 int);
+sql create table zeee22 (ts timestamp, f1 int);
+sql show create database db4;
+sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db4;
+ 
+sql drop database if exists db5;
+sql create database db5 vgroups 5 TABLE_PREFIX 1;
+sql use db5;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table baaa11 (ts timestamp, f1 int);
+sql create table caaa11 (ts timestamp, f1 int);
+sql create table daaa11 (ts timestamp, f1 int);
+sql create table faaa11 (ts timestamp, f1 int);
+sql create table gbbb11 (ts timestamp, f1 int);
+sql create table hbbb11 (ts timestamp, f1 int);
+sql create table ibbb11 (ts timestamp, f1 int);
+sql create table jbbb11 (ts timestamp, f1 int);
+sql show create database db5;
+sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db5;
+ 
+sql drop database if exists db6;
+sql create database db6 vgroups 5 TABLE_SUFFIX 2;
+sql use db6;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table taaa12 (ts timestamp, f1 int);
+sql create table taaa13 (ts timestamp, f1 int);
+sql create table taaa14 (ts timestamp, f1 int);
+sql create table tbbb23 (ts timestamp, f1 int);
+sql create table tbbb24 (ts timestamp, f1 int);
+sql create table tbbb31 (ts timestamp, f1 int);
+sql create table tbbb32 (ts timestamp, f1 int);
+sql create table tbbb33 (ts timestamp, f1 int);
+sql show create database db6;
+sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db6;
+ 
+sql drop database if exists db7;
+sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92;
+sql use db7;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table taaa12 (ts timestamp, f1 int);
+sql create table taaa13 (ts timestamp, f1 int);
+sql create table tbbb21 (ts timestamp, f1 int);
+sql create table tbbb22 (ts timestamp, f1 int);
+sql create table tbbb23 (ts timestamp, f1 int);
+sql create table tbbb24 (ts timestamp, f1 int);
+sql create table tccc31 (ts timestamp, f1 int);
+sql create table tccc32 (ts timestamp, f1 int);
+sql create table tccc33 (ts timestamp, f1 int);
+sql create table tddd24 (ts timestamp, f1 int);
+sql create table tddd31 (ts timestamp, f1 int);
+sql create table tddd32 (ts timestamp, f1 int);
+sql create table tddd33 (ts timestamp, f1 int);
+sql show create database db7;
+sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a;
+sql drop database if exists db7;
+
+sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2;
+sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192;
+sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191;
+sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92;
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/query/sys_tbname.sim

@@ -109,5 +109,26 @@ if $rows != 5000 then
         return -1
 endi
 
+sql create database d1;
+sql create stable d1.st1 (ts timestamp, f int) tags(t int);
+sql create stable d1.st2 (ts timestamp, f int) tags(t int);
+sql create table d1.ct1 using d1.st1 tags(1);
+sql create table d1.ct2 using d1.st2 tags(2);
 
+sql create database d2;
+sql create stable d2.st1(ts timestamp, f int) tags(t int);
+sql create stable d2.st2(ts timestamp, f int) tags(t int);
+sql create table d2.ct1 using d2.st1 tags(1);
+sql create table d2.ct2 using d2.st2 tags(2);
+
+sql create database d3;
+sql create stable d3.st1(ts timestamp, f int) tags(t int);
+sql create stable d3.st2(ts timestamp, f int) tags(t int);
+sql create table d3.ct1 using d3.st1 tags(1);
+sql create table d3.ct2 using d3.st2 tags(2);
+sql select count(*), stable_name, db_name from information_schema.ins_tables where db_name != 'd2' group by stable_name,db_name
+print $rows
+if $rows != 9 then
+  return -1
+endi
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/query/tableCount.sim

@@ -104,4 +104,9 @@ if $data62 != 5 then
   return -1
 endi
 
+sql select distinct db_name from information_schema.ins_tables;
+print $rows
+if $rows != 4 then
+  return -1
+endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/basic1.sim

@@ -37,7 +37,7 @@ if $loop_count == 20 then
 endi
 
 if $rows != 4 then
-  print =====rows=$rows, expect 4
+  print =====rows=$rows  expect 4
   goto loop0
 endi
 

tests/script/tsim/tag/bigint.sim

@@ -123,6 +123,17 @@ sql select * from $mt where tgcol = 1
 if $rows != 100 then
   return -1
 endi
+
+sql select * from $mt where tgcol = '1'
+if $rows != 100 then
+  return -1
+endi
+
+sql select * from $mt where tgcol = "1"
+if $rows != 100 then
+  return -1
+endi
+
 sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1

tests/script/tsim/tag/double.sim

@@ -123,6 +123,15 @@ sql select * from $mt where tgcol = 1
 if $rows != 100 then
   return -1
 endi
+
+sql select * from $mt where tgcol = '1';
+if $rows != 100 then
+  return -1
+endi
+sql select * from $mt where tgcol = "1.0"
+if $rows != 100 then
+  return -1
+endi
 sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1

tests/script/tsim/tag/float.sim

@@ -123,6 +123,16 @@ sql select * from $mt where tgcol = 1
 if $rows != 100 then
   return -1
 endi
+
+sql select * from $mt where tgcol = "1.0"
+if $rows != 100 then
+  return -1
+endi
+
+sql select * from $mt where tgcol = "1"
+if $rows != 100 then
+  return -1
+endi
 sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1

tests/script/tsim/tag/int.sim

@@ -123,6 +123,16 @@ sql select * from $mt where tgcol = 1
 if $rows != 100 then
   return -1
 endi
+
+sql select * from $mt where tgcol = '1'
+if $rows != 100 then
+  return -1
+endi
+
+sql select * from $mt where tgcol = "1";
+if $rows != 100 then
+  return -1
+endi
 sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1

tests/script/tsim/tag/int_float.sim

@@ -85,10 +85,22 @@ sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1
 endi
+
 sql select * from $mt where tgcol = 1
 if $rows != 100 then
   return -1
 endi
+
+sql select * from $mt where tgcol = '1'
+if $rows != 100 then
+  return -1
+endi
+
+sql select * from $mt where tgcol = "1"
+if $rows != 100 then
+  return -1
+endi
+
 sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1

tests/script/tsim/tag/tinyint.sim

@@ -115,14 +115,36 @@ sql select * from $mt where tgcol = 0
 if $rows != 100 then
   return -1
 endi
+
+sql select * from $mt where tgcol = '0'
+if $rows != 100 then
+  return -1
+endi
+
+sql select * from $mt where tgcol = "0"
+if $rows != 100 then
+  return -1
+endi
 sql select * from $mt where tgcol <> 0
 if $rows != 100 then
   return -1
 endi
+
 sql select * from $mt where tgcol = 1
 if $rows != 100 then
   return -1
 endi
+sql select * from $mt where tgcol = "1"
+if $rows != 100 then
+  return -1
+endi
+
+sql select * from $mt where tgcol = '1'
+if $rows != 100 then
+  return -1
+endi
+
+
 sql select * from $mt where tgcol <> 1
 if $rows != 100 then
   return -1

tests/script/tsim/user/privilege_table.sim

@@ -0,0 +1,302 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print =============== init env
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create stable st1(ts timestamp, i int) tags(id int, loc varchar(20));
+sql create table st1s1 using st1 tags(1, 'beijing');
+sql create table st1s2 using st1 tags(2, 'shanghai');
+sql insert into st1s1 values(now, 1) st1s2 values(now, 2);
+sql create stable st2(ts timestamp, i int) tags(id int, loc varchar(20));
+sql create table st2s1 using st2 tags(1, 'beijing');
+sql create table st2s2 using st2 tags(2, 'shanghai');
+sql insert into st2s1 values(now, 1) st2s2 values(now, 2);
+sql create user wxy pass 'taosdata';
+
+print =============== case 1: database unauthorized and table unauthorized
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql_error select * from test.st1;
+sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+
+print =============== case 2: database unauthorized and table read privilege
+sql close
+sql connect
+
+sql grant read on test.st1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1;
+if $rows != 2 then 
+  return -1
+endi
+sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 3: database unauthorized and table read privilege with condition
+sql close
+sql connect
+
+sql revoke read on test.st1 from wxy;
+sql grant read on test.st1 with id = 1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1; 
+if $rows != 1 then 
+  return -1
+endi
+sql_error insert into test.st1s1 values(now, 10);
+sql_error insert into test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 4: database unauthorized and table write privilege
+sql close
+sql connect
+
+sql revoke read on test.st1 with id = 1 from wxy;
+sql grant write on test.st1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql_error select tbname, * from test.st1; 
+sql insert into test.st1s1 values(now, 10);
+sql insert into test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 5: database unauthorized and table write privilege with condition
+sql close
+sql connect
+
+sql revoke write on test.st1 from wxy;
+sql grant write on test.st1 with id = 1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql_error select tbname, * from test.st1; 
+sql insert into test.st1s1 values(now, 10);
+sql insert into test.st1s3 using test.st1 tags(1, 'dachang') values(now, 100);
+sql_error insert into test.st1s2 values(now, 20);
+sql_error insert into test.st1s4 using test.st1 tags(3, 'dachang') values(now, 300);
+sql_error select * from test.st2; 
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 6: database read privilege and table unauthorized
+sql close
+sql connect
+
+sql revoke write on test.st1 with id = 1 from wxy;
+sql grant read on test.* to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1;
+if $rows != 6 then 
+  return -1
+endi
+sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql select * from test.st2; 
+if $rows != 2 then 
+  return -1
+endi
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 7: database read privilege and table read privilege
+sql close
+sql connect
+
+sql grant read on test.st1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1; 
+if $rows != 6 then 
+  return -1
+endi
+sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql select * from test.st2; 
+if $rows != 2 then 
+  return -1
+endi
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 8: database read privilege and table read privilege with condition
+sql close
+sql connect
+
+sql revoke read on test.st1 from wxy;
+sql grant read on test.st1 with id = 1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1; 
+if $rows != 4 then 
+  return -1
+endi
+sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql select * from test.st2; 
+if $rows != 2 then 
+  return -1
+endi
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 9: database read privilege and table write privilege
+sql close
+sql connect
+
+sql revoke read on test.st1 with id = 1 from wxy;
+sql grant write on test.st1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1;
+if $rows != 6 then 
+  return -1
+endi
+sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql select * from test.st2;
+if $rows != 2 then 
+  return -1
+endi
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 10: database read privilege and table write privilege with condition
+sql close
+sql connect
+
+sql revoke write on test.st1 from wxy;
+sql grant write on test.st1 with id = 1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1;
+if $rows != 8 then 
+  return -1
+endi
+sql insert into test.st1s1 values(now, 10);
+sql_error insert into test.st1s2 values(now, 20);
+sql select * from test.st2;
+if $rows != 2 then 
+  return -1
+endi
+sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 11: database write privilege and table unauthorized
+sql close
+sql connect
+
+sql revoke read on test.* from wxy;
+sql revoke write on test.st1 with id = 1 from wxy;
+sql grant write on test.* to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql_error select * from test.st1; 
+sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 12: database write privilege and table read privilege
+sql close
+sql connect
+
+sql grant read on test.st1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1; 
+if $rows != 11 then 
+  return -1
+endi
+sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 13: database write privilege and table read privilege with condition
+sql close
+sql connect
+
+sql revoke read on test.st1 from wxy;
+sql grant read on test.st1 with id = 1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql select * from test.st1; 
+if $rows != 8 then 
+  return -1
+endi
+sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 14: database write privilege and table write privilege
+sql close
+sql connect
+
+sql revoke read on test.st1 with id = 1 from wxy;
+sql grant write on test.st1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql_error select * from test.st1; 
+sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+print =============== case 15: database write privilege and table write privilege with condition
+sql close
+sql connect
+
+sql revoke write on test.st1 from wxy;
+sql grant write on test.st1 with id = 1 to wxy;
+
+sql close
+sql connect wxy
+
+sql reset query cache;
+sql_error select * from test.st1; 
+sql insert into test.st1s1 values(now, 10);
+sql_error insert into test.st1s2 values(now, 20);
+sql_error select * from test.st2; 
+sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20);
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/win-test-file

@@ -3,6 +3,7 @@
 ./test.sh -f tsim/user/privilege_db.sim
 ./test.sh -f tsim/user/privilege_sysinfo.sim
 ./test.sh -f tsim/user/privilege_topic.sim
+./test.sh -f tsim/user/privilege_table.sim
 ./test.sh -f tsim/db/alter_option.sim
 rem ./test.sh -f tsim/db/alter_replica_13.sim
 ./test.sh -f tsim/db/alter_replica_31.sim