merge:from 3.0

cmake/cmake.define

@@ -46,7 +46,7 @@ ENDIF ()
 
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
-    SET(COMMON_FLAGS "/w /D_WIN32")
+    SET(COMMON_FLAGS "/w /D_WIN32 /Zi")
     SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
     # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
     #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
@@ -71,8 +71,8 @@ ELSE ()
     ENDIF ()
 
     IF (${SANITIZER} MATCHES "true")
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -g3")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -g3")
 	MESSAGE(STATUS "Will compile with Address Sanitizer!")
     ELSE ()
         SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")

contrib/CMakeLists.txt

@@ -100,8 +100,10 @@ endif(${BUILD_WITH_NURAFT})
 
 # addr2line
 if(${BUILD_ADDR2LINE})
+    if(NOT ${TD_WINDOWS})
         cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
         cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    endif(NOT ${TD_WINDOWS})
 endif(${BUILD_ADDR2LINE})
 
 # download dependencies
@@ -335,6 +337,7 @@ endif(${BUILD_WITH_SQLITE})
 
 # addr2line
 if(${BUILD_ADDR2LINE})
+    if(NOT ${TD_WINDOWS})    
         check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H) 
         check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H ) 
         check_include_file( "inttypes.h"      HAVE_INTTYPES_H   ) 
@@ -374,6 +377,7 @@ if(${BUILD_ADDR2LINE})
         add_library(addr2line STATIC "addr2line/addr2line.c")
         target_link_libraries(addr2line PUBLIC libdwarf dl z)
         target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" )
+    endif(NOT ${TD_WINDOWS})
 endif(${BUILD_ADDR2LINE})
 
 

docs-cn/14-reference/03-connector/csharp.mdx

@@ -18,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx"
 
 `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。
 
-`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) 文档自行编写。
+`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动（taosc）建立与 TDengine 运行实例的连接，提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式，用户可以参考 [REST API](/reference/rest-api/) 文档自行编写。
 
 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`，并通过 `TDengine.Connector` 连接 TDengine 集群，进行数据写入、查询等基本操作。
 

docs-en/14-reference/03-connector/csharp.mdx

@@ -19,7 +19,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx"
 
 `TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data.
 
-The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) documentation.
+The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [REST API](/reference/rest-api/) documentation.
 
 This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying.
 

example/src/tstream.c

@@ -82,9 +82,7 @@ int32_t create_stream() {
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
   /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
   pRes = taos_query(
-      pConn,
+      pConn, "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from tu1 interval(10m)");
-      "create stream stream1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k "
-      "from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;

include/common/tmsg.h

@@ -244,12 +244,12 @@ typedef struct {
   const void* pMsg;
 } SSubmitMsgIter;
 
-int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter);
+int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter);
 int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
 int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
 STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
 // for debug
-int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema);
+int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema);
 
 typedef struct {
   int32_t code;
@@ -1697,7 +1697,7 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam);
 
 // TDMT_VND_CREATE_STB ==============
 typedef struct SVCreateStbReq {
-  const char*    name;
+  char*          name;
   tb_uid_t       suid;
   int8_t         rollup;
   SSchemaWrapper schema;
@@ -1710,7 +1710,7 @@ int tDecodeSVCreateStbReq(SDecoder* pCoder, SVCreateStbReq* pReq);
 
 // TDMT_VND_DROP_STB ==============
 typedef struct SVDropStbReq {
-  const char* name;
+  char*    name;
   tb_uid_t suid;
 } SVDropStbReq;
 
@@ -1723,13 +1723,13 @@ typedef struct SVCreateTbReq {
   int32_t  flags;
   tb_uid_t uid;
   int64_t  ctime;
-  const char* name;
+  char*    name;
   int32_t  ttl;
   int8_t   type;
   union {
     struct {
       tb_uid_t suid;
-      const uint8_t* pTag;
+      uint8_t* pTag;
     } ctb;
     struct {
       SSchemaWrapper schema;
@@ -1777,7 +1777,7 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc
 
 // TDMT_VND_DROP_TABLE =================
 typedef struct {
-  const char* name;
+  char*  name;
   int8_t igNotExists;
 } SVDropTbReq;
 
@@ -1809,9 +1809,9 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
 
 // TDMT_VND_ALTER_TABLE =====================
 typedef struct {
-  const char* tbName;
+  char*  tbName;
   int8_t action;
-  const char* colName;
+  char*  colName;
   // TSDB_ALTER_TABLE_ADD_COLUMN
   int8_t  type;
   int8_t  flags;
@@ -1820,17 +1820,17 @@ typedef struct {
   // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
   int32_t colModBytes;
   // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
-  const char* colNewName;
+  char* colNewName;
   // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
-  const char*    tagName;
+  char*    tagName;
   int8_t   isNull;
   uint32_t nTagVal;
-  const uint8_t* pTagVal;
+  uint8_t* pTagVal;
   // TSDB_ALTER_TABLE_UPDATE_OPTIONS
   int8_t  updateTTL;
   int32_t newTTL;
   int8_t  updateComment;
-  const char* newComment;
+  char*   newComment;
 } SVAlterTbReq;
 
 int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@@ -2267,8 +2267,8 @@ typedef struct {
   int64_t  interval;
   int64_t  offset;  // use unit by precision of DB
   int64_t  sliding;
-  const char* expr;  // sma expression
+  char*    expr;  // sma expression
-  const char* tagsFilter;
+  char*    tagsFilter;
 } STSma;  // Time-range-wise SMA
 
 typedef STSma SVCreateTSmaReq;
@@ -2600,7 +2600,7 @@ typedef struct {
   int64_t       uid;
   int32_t       sver;
   uint32_t      nData;
-  const uint8_t* pData;
+  uint8_t*      pData;
   SVCreateTbReq cTbReq;
 } SVSubmitBlk;
 

include/libs/index/index.h

@@ -16,9 +16,11 @@
 #ifndef _TD_INDEX_H_
 #define _TD_INDEX_H_
 
+#include "nodes.h"
 #include "os.h"
 #include "taoserror.h"
 #include "tarray.h"
+#include "tglobal.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -189,6 +191,12 @@ void        indexTermDestroy(SIndexTerm* p);
  */
 void indexInit();
 
+/* index filter */
+typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
+
+SIdxFltStatus idxGetFltStatus(SNode* pFilterNode);
+
+int32_t doFilterTag(const SNode* pFilterNode, SArray* result);
 /*
  * destory index env
  *

include/libs/nodes/nodes.h

@@ -61,8 +61,8 @@ extern "C" {
 
 #define DESTORY_LIST(list)    \
   do {                        \
-    nodesDestroyList(list); \
+    nodesDestroyList((list)); \
-    list = NULL;            \
+    (list) = NULL;            \
   } while (0)
 
 typedef enum ENodeType {
@@ -96,6 +96,7 @@ typedef enum ENodeType {
   QUERY_NODE_EXPLAIN_OPTIONS,
   QUERY_NODE_STREAM_OPTIONS,
   QUERY_NODE_TOPIC_OPTIONS,
+  QUERY_NODE_LEFT_VALUE,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,

include/libs/nodes/querynodes.h

@@ -81,6 +81,7 @@ typedef struct SValueNode {
   char*     literal;
   bool      isDuration;
   bool      translate;
+  bool      notReserved;
   int16_t   placeholderNo;
   union {
     bool     b;
@@ -93,6 +94,10 @@ typedef struct SValueNode {
   char    unit;
 } SValueNode;
 
+typedef struct SLeftValueNode {
+  ENodeType type;
+} SLeftValueNode;
+
 typedef struct SOperatorNode {
   SExprNode     node;  // QUERY_NODE_OPERATOR
   EOperatorType opType;

include/libs/stream/tstream.h

@@ -114,17 +114,12 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit)
 int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
 void*   streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
 
-typedef struct {
-  void* inputHandle;
-  void* executor;
-} SStreamRunner;
-
 typedef struct {
   int8_t parallelizable;
   char*  qmsg;
   // followings are not applicable to encoder and decoder
-  int8_t         numOfRunners;
+  void* inputHandle;
-  SStreamRunner* runners;
+  void* executor;
 } STaskExec;
 
 typedef struct {
@@ -320,17 +315,15 @@ int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
 int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
 int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
 
-int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
-
 int32_t streamTaskRun(SStreamTask* pTask);
 
 int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
 
 int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb);
-int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
+int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
-int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp);
+int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp);
-int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
+int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
-int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
+int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -132,6 +132,7 @@ typedef enum EOperatorType {
   OP_TYPE_MOD,
   // unary arithmetic operator
   OP_TYPE_MINUS,
+  OP_TYPE_ASSIGN,
 
   // bit operator
   OP_TYPE_BIT_AND,

include/util/tencode.h

@@ -39,7 +39,7 @@ typedef struct {
 } SEncoder;
 
 typedef struct {
-  const uint8_t* data;
+  uint8_t*      data;
   uint32_t      size;
   uint32_t      pos;
   SCoderMem*    mList;
@@ -120,7 +120,7 @@ static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t le
 static int32_t tEncodeCStr(SEncoder* pCoder, const char* val);
 
 /* ------------------------ DECODE ------------------------ */
-void           tDecoderInit(SDecoder* pCoder, const uint8_t* data, uint32_t size);
+void           tDecoderInit(SDecoder* pCoder, uint8_t* data, uint32_t size);
 void           tDecoderClear(SDecoder* SDecoder);
 int32_t        tStartDecode(SDecoder* pCoder);
 void           tEndDecode(SDecoder* pCoder);
@@ -141,9 +141,9 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val);
 static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val);
 static int32_t tDecodeFloat(SDecoder* pCoder, float* val);
 static int32_t tDecodeDouble(SDecoder* pCoder, double* val);
-static int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len);
+static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len);
-static int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len);
+static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len);
-static int32_t tDecodeCStr(SDecoder* pCoder, const char** val);
+static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
 static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
 
 /* ------------------------ IMPL ------------------------ */
@@ -317,7 +317,7 @@ static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
   if (tDecodeU16v(pCoder, &tval) < 0) {
     return -1;
   }
-  *val = ZIGZAGD(int16_t, tval);
+  if (val) *val = ZIGZAGD(int16_t, tval);
   return 0;
 }
 
@@ -331,7 +331,7 @@ static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
   if (tDecodeU32v(pCoder, &tval) < 0) {
     return -1;
   }
-  *val = ZIGZAGD(int32_t, tval);
+  if (val) *val = ZIGZAGD(int32_t, tval);
   return 0;
 }
 
@@ -345,7 +345,7 @@ static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
   if (tDecodeU64v(pCoder, &tval) < 0) {
     return -1;
   }
-  *val = ZIGZAGD(int64_t, tval);
+  if (val) *val = ZIGZAGD(int64_t, tval);
   return 0;
 }
 
@@ -377,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len) {
+static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
   if (tDecodeU32v(pCoder, len) < 0) return -1;
 
   if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1;
@@ -389,19 +389,19 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val,
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len) {
+static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
-  if (tDecodeBinary(pCoder, (const uint8_t**)val, len) < 0) return -1;
+  if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1;
   (*len) -= 1;
   return 0;
 }
 
-static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, const char** val) {
+static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
   uint32_t len;
   return tDecodeCStrAndLen(pCoder, val, &len);
 }
 
 static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
-  const char* pStr;
+  char*    pStr;
   uint32_t len;
   if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1;
 

packaging/tools/install.sh

@@ -6,16 +6,36 @@
 set -e
 #set -x
 
+verMode=edge
+pagMode=full
+
+iplist=""
+serverFqdn=""
+
 # -----------------------Variables definition---------------------
 script_dir=$(dirname $(readlink -f "$0"))
 # Dynamic directory
-data_dir="/var/lib/taos"
-log_dir="/var/log/taos"
 
-data_link_dir="/usr/local/taos/data"
+clientName="taos"
-log_link_dir="/usr/local/taos/log"
+serverName="taosd"
+configFile="taos.cfg"
+productName="TDengine"
+emailName="taosdata.com"
+uninstallScript="rmtaos"
+historyFile="taos_history"
+tarName="taos.tar.gz"
+dataDir="/var/lib/taos"
+logDir="/var/log/taos"
+configDir="/etc/taos"
+installDir="/usr/local/taos"
+adapterName="taosadapter"
+benchmarkName="taosBenchmark"
+dumpName="taosdump"
+demoName="taosdemo"
 
-cfg_install_dir="/etc/taos"
+data_dir=${dataDir}
+log_dir=${logDir}
+cfg_install_dir=${configDir}
 
 bin_link_dir="/usr/bin"
 lib_link_dir="/usr/lib"
@@ -23,21 +43,13 @@ lib64_link_dir="/usr/lib64"
 inc_link_dir="/usr/include"
 
 #install main path
-install_main_dir="/usr/local/taos"
+install_main_dir=${installDir}
-
 # old bin dir
-bin_dir="/usr/local/taos/bin"
+bin_dir="${installDir}/bin"
 
 service_config_dir="/etc/systemd/system"
-
+nginx_port=6060
-#taos-tools para
+nginx_dir="/usr/local/nginxd"
-demoName="taosdemo"
-benchmarkName="taosBenchmark"
-dumpName="taosdump"
-emailName="taosdata.com"
-taosName="taos"
-toolsName="taostools"
-
 
 # Color setting
 RED='\033[0;31m'
@@ -74,7 +86,6 @@ else
   service_mod=2
 fi
 
-
 # get the operating system type for using the corresponding init file
 # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification
 #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
@@ -110,11 +121,10 @@ else
   echo " osinfo: ${osinfo}"
   echo " This is an officially unverified linux system,"
   echo " if there are any problems with the installation and operation, "
-  echo " please feel free to contact taosdata.com for support."
+  echo " please feel free to contact ${emailName} for support."
   os_type=1
 fi
 
-
 # =============================  get input parameters =================================================
 
 # install.sh -v [server | client]  -e [yes | no] -i [systemd | service | ...]
@@ -124,8 +134,7 @@ interactiveFqdn=yes   # [yes | no]
 verType=server      # [server | client]
 initType=systemd    # [systemd | service | ...]
 
-while getopts "hv:e:i:" arg
+while getopts "hv:e:i:" arg; do
-do
   case $arg in
   e)
     #echo "interactiveFqdn=$OPTARG"
@@ -140,7 +149,7 @@ do
     initType=$(echo $OPTARG)
     ;;
   h)
-      echo "Usage: `basename $0` -v [server | client]  -e [yes | no]"
+    echo "Usage: $(basename $0) -v [server | client]  -e [yes | no]"
     exit 0
     ;;
   ?) #unknow option
@@ -165,11 +174,11 @@ function install_main_path() {
   ${csudo}mkdir -p ${install_main_dir}
   ${csudo}mkdir -p ${install_main_dir}/cfg
   ${csudo}mkdir -p ${install_main_dir}/bin
-    ${csudo} mkdir -p ${install_main_dir}/connector
+  #  ${csudo}mkdir -p ${install_main_dir}/connector
-    ${csudo} mkdir -p ${install_main_dir}/lib
+  ${csudo}mkdir -p ${install_main_dir}/driver
   ${csudo}mkdir -p ${install_main_dir}/examples
   ${csudo}mkdir -p ${install_main_dir}/include
-    ${csudo} mkdir -p ${install_main_dir}/init.d
+  #  ${csudo}mkdir -p ${install_main_dir}/init.d
   if [ "$verMode" == "cluster" ]; then
     ${csudo}mkdir -p ${nginx_dir}
   fi
@@ -181,74 +190,139 @@ function install_main_path() {
 
 function install_bin() {
   # Remove links
-    ${csudo} rm -f ${bin_link_dir}/taos     || :
+  ${csudo}rm -f ${bin_link_dir}/${clientName} || :
-    ${csudo} rm -f ${bin_link_dir}/taosd    || :
+  ${csudo}rm -f ${bin_link_dir}/${serverName} || :
-    ${csudo} rm -f ${bin_link_dir}/taosadapter     || :
+  ${csudo}rm -f ${bin_link_dir}/${adapterName} || :
-    ${csudo} rm -f ${bin_link_dir}/create_table || :
+  ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
-    ${csudo} rm -f ${bin_link_dir}/tmq_sim   || :
+  ${csudo}rm -f ${bin_link_dir}/tarbitrator || :
-    ${csudo} rm -f ${bin_link_dir}/taosdump || :
+  ${csudo}rm -f ${bin_link_dir}/set_core || :
-    ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
+  ${csudo}rm -f ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || :
-    #${csudo} rm -f ${bin_link_dir}/set_core   || :
+  ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
 
   ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/*
 
   #Make link
-    [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos                          || :
+  [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
-    [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd                       || :
+  [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
-    [ -x ${install_main_dir}/bin/create_table ] && ${csudo} ln -s ${install_main_dir}/bin/create_table ${bin_link_dir}/create_table   || :
+  [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
-    [ -x ${install_main_dir}/bin/tmq_sim ] && ${csudo} ln -s ${install_main_dir}/bin/tmq_sim ${bin_link_dir}/tmq_sim     || :
+  [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
-#    [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo              || :
+  [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
-#    [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump              || :
+  [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
-    [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos              || :
+  [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
-#    [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core        || :
+  [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
+  [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
+  [ -x ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ] && ${csudo}ln -s ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || :
+  [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || :
+
+  if [ "$verMode" == "cluster" ]; then
+    ${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/*
+    ${csudo}mkdir -p ${nginx_dir}/logs
+    ${csudo}chmod 777 ${nginx_dir}/sbin/nginx
+  fi
 }
 
 function install_lib() {
   # Remove links
   ${csudo}rm -f ${lib_link_dir}/libtaos.* || :
   ${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
-    ${csudo} rm -f ${lib_link_dir}/libtdb.*         || :
+  #${csudo}rm -rf ${v15_java_app_dir}              || :
-    ${csudo} rm -f ${lib64_link_dir}/libtdb.*       || :
+  ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/*
 
-    ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/*
+  ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
-
-    ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1
   ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
 
   if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
-      ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1     || :
+    ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
     ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
   fi
 
   ${csudo}ldconfig
 }
 
+function install_avro() {
+  if [ "$osType" != "Darwin" ]; then
+    avro_dir=${script_dir}/avro
+    if [ -f "${avro_dir}/lib/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then
+      ${csudo}/usr/bin/install -c -d /usr/local/$1
+      ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.so.23.0.0 /usr/local/$1
+      ${csudo}ln -sf /usr/local/$1/libavro.so.23.0.0 /usr/local/$1/libavro.so.23
+      ${csudo}ln -sf /usr/local/$1/libavro.so.23 /usr/local/$1/libavro.so
+
+      ${csudo}/usr/bin/install -c -d /usr/local/$1
+      [ -f ${avro_dir}/lib/libavro.a ] &&
+        ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.a /usr/local/$1
+
+      if [ -d /etc/ld.so.conf.d ]; then
+        echo "/usr/local/$1" | ${csudo}tee /etc/ld.so.conf.d/libavro.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/libavro.conf"
+        ${csudo}ldconfig
+      else
+        echo "/etc/ld.so.conf.d not found!"
+      fi
+    fi
+  fi
+}
+
+function install_jemalloc() {
+  jemalloc_dir=${script_dir}/jemalloc
+
+  if [ -d ${jemalloc_dir} ]; then
+    ${csudo}/usr/bin/install -c -d /usr/local/bin
+
+    if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then
+      ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin
+    fi
+    if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then
+      ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin
+    fi
+    if [ -f ${jemalloc_dir}/bin/jeprof ]; then
+      ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin
+    fi
+    if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then
+      ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc
+      ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc
+    fi
+    if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then
+      ${csudo}/usr/bin/install -c -d /usr/local/lib
+      ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
+      ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+      ${csudo}/usr/bin/install -c -d /usr/local/lib
+      if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
+        ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
+      fi
+      if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+        ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
+      fi
+      if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+        ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
+        ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
+      fi
+    fi
+    if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then
+      ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc
+      ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc
+    fi
+    if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then
+      ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3
+      ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
+    fi
+
+    if [ -d /etc/ld.so.conf.d ]; then
+      echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf"
+      ${csudo}ldconfig
+    else
+      echo "/etc/ld.so.conf.d not found!"
+    fi
+  fi
+}
+
 function install_header() {
   ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || :
   ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/*
   ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
-#    ${csudo} ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
+  ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h
   ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
 }
 
-# temp install taosBenchmark
-function install_taosTools() {
-    ${csudo} rm -f ${bin_link_dir}/${benchmarkName}    || :
-    ${csudo} rm -f ${bin_link_dir}/${dumpName}         || :
-    ${csudo} rm -f ${bin_link_dir}/rm${toolsName}      || :
-
-    ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName}
-    ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName}
-    ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName}
-    #Make link
-    [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \
-        ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName}        || :
-    [[ -x ${install_main_dir}/bin/${demoName} ]] && \
-        ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName}                  || :
-    [[ -x ${install_main_dir}/bin/${dumpName} ]] && \
-        ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName}                  || :
-}
-
 function add_newHostname_to_hosts() {
   localIp="127.0.0.1"
   OLD_IFS="$IFS"
@@ -256,8 +330,7 @@ function add_newHostname_to_hosts() {
   iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
   arr=($iphost)
   IFS="$OLD_IFS"
-  for s in "${arr[@]}"
+  for s in "${arr[@]}"; do
-  do
     if [[ "$s" == "$localIp" ]]; then
       return
     fi
@@ -277,15 +350,12 @@ function set_hostname() {
   done
 
   ${csudo}hostname $newHostname || :
-  retval=`echo $?`
+  retval=$(echo $?)
   if [[ $retval != 0 ]]; then
     echo
     echo "set hostname fail!"
     return
   fi
-  #echo -e -n "$(hostnamectl status --static)"
-  #echo -e -n "$(hostnamectl status --transient)"
-  #echo -e -n "$(hostnamectl status --pretty)"
 
   #ubuntu/centos /etc/hostname
   if [[ -e /etc/hostname ]]; then
@@ -297,7 +367,7 @@ function set_hostname() {
     ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || :
   fi
 
-  ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg
+  ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile}
   serverFqdn=$newHostname
 
   if [[ -e /etc/hosts ]]; then
@@ -311,8 +381,7 @@ function is_correct_ipaddr() {
   IFS=" "
   arr=($iplist)
   IFS="$OLD_IFS"
-  for s in "${arr[@]}"
+  for s in "${arr[@]}"; do
-  do
     if [[ "$s" == "$newIp" ]]; then
       return 0
     fi
@@ -332,7 +401,7 @@ function set_ipAsFqdn() {
     echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}"
     localFqdn="127.0.0.1"
     # Write the local FQDN to configuration file
-    ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
+    ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile}
     serverFqdn=$localFqdn
     echo
     return
@@ -349,12 +418,12 @@ function set_ipAsFqdn() {
     if [ ! -z "$localFqdn" ]; then
       # Check if correct ip address
       is_correct_ipaddr $localFqdn
-        retval=`echo $?`
+      retval=$(echo $?)
       if [[ $retval != 0 ]]; then
         read -p "Please choose an IP from local IP list:" localFqdn
       else
         # Write the local FQDN to configuration file
-          ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg
+        ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile}
         serverFqdn=$localFqdn
         break
       fi
@@ -373,8 +442,7 @@ function local_fqdn_check() {
     echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}"
     echo
 
-    while true
+    while true; do
-    do
       read -r -p "Set hostname now? [Y/n] " input
       if [ ! -n "$input" ]; then
         set_hostname
@@ -400,6 +468,74 @@ function local_fqdn_check() {
   fi
 }
 
+function install_adapter_config() {
+  if [ ! -f "${cfg_install_dir}/${adapterName}.toml" ]; then
+    ${csudo}mkdir -p ${cfg_install_dir}
+    [ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}
+    [ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/${adapterName}.toml
+  fi
+
+  [ -f ${script_dir}/cfg/${adapterName}.toml ] &&
+    ${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new
+
+  [ -f ${cfg_install_dir}/${adapterName}.toml ] &&
+    ${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml
+
+  [ ! -z $1 ] && return 0 || : # only install client
+
+}
+
+function install_config() {
+
+  if [ ! -f "${cfg_install_dir}/${configFile}" ]; then
+    ${csudo}mkdir -p ${cfg_install_dir}
+    [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir}
+    ${csudo}chmod 644 ${cfg_install_dir}/*
+  fi
+
+  ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new
+  ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg
+
+  [ ! -z $1 ] && return 0 || : # only install client
+
+  if ((${update_flag} == 1)); then
+    return 0
+  fi
+
+  if [ "$interactiveFqdn" == "no" ]; then
+    return 0
+  fi
+
+  local_fqdn_check
+
+  echo
+  echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}"
+  echo
+  echo -e -n "${GREEN}OR leave it blank to build one${NC}:"
+  read firstEp
+  while true; do
+    if [ ! -z "$firstEp" ]; then
+      ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile}
+      break
+    else
+      break
+    fi
+  done
+
+  echo
+  echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: "
+  read emailAddr
+  while true; do
+    if [ ! -z "$emailAddr" ]; then
+      email_file="${install_main_dir}/email"
+      ${csudo}bash -c "echo $emailAddr > ${email_file}"
+      break
+    else
+      break
+    fi
+  done
+}
+
 function install_log() {
   ${csudo}rm -rf ${log_dir} || :
   ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
@@ -413,13 +549,96 @@ function install_data() {
   ${csudo}ln -s ${data_dir} ${install_main_dir}/data
 }
 
-function clean_service_on_systemd() {
+function install_connector() {
-    taosd_service_config="${service_config_dir}/taosd.service"
+  [ -d "${script_dir}/connector/" ] && ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/
-    if systemctl is-active --quiet taosd; then
+}
-        echo "TDengine is running, stopping it..."
+
-        ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null
+function install_examples() {
+  if [ -d ${script_dir}/examples ]; then
+    ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples
   fi
-    ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null
+}
+
+function clean_service_on_sysvinit() {
+  if pidof ${serverName} &>/dev/null; then
+    ${csudo}service ${serverName} stop || :
+  fi
+
+  if pidof tarbitrator &>/dev/null; then
+    ${csudo}service tarbitratord stop || :
+  fi
+
+  if ((${initd_mod} == 1)); then
+    if [ -e ${service_config_dir}/${serverName} ]; then
+      ${csudo}chkconfig --del ${serverName} || :
+    fi
+
+    if [ -e ${service_config_dir}/tarbitratord ]; then
+      ${csudo}chkconfig --del tarbitratord || :
+    fi
+  elif ((${initd_mod} == 2)); then
+    if [ -e ${service_config_dir}/${serverName} ]; then
+      ${csudo}insserv -r ${serverName} || :
+    fi
+    if [ -e ${service_config_dir}/tarbitratord ]; then
+      ${csudo}insserv -r tarbitratord || :
+    fi
+  elif ((${initd_mod} == 3)); then
+    if [ -e ${service_config_dir}/${serverName} ]; then
+      ${csudo}update-rc.d -f ${serverName} remove || :
+    fi
+    if [ -e ${service_config_dir}/tarbitratord ]; then
+      ${csudo}update-rc.d -f tarbitratord remove || :
+    fi
+  fi
+
+  ${csudo}rm -f ${service_config_dir}/${serverName} || :
+  ${csudo}rm -f ${service_config_dir}/tarbitratord || :
+
+  if $(which init &>/dev/null); then
+    ${csudo}init q || :
+  fi
+}
+
+function install_service_on_sysvinit() {
+  clean_service_on_sysvinit
+  sleep 1
+
+  if ((${os_type} == 1)); then
+    #    ${csudo}cp -f ${script_dir}/init.d/${serverName}.deb ${install_main_dir}/init.d/${serverName}
+    ${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName}
+    #    ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord
+    ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
+  elif ((${os_type} == 2)); then
+    #    ${csudo}cp -f ${script_dir}/init.d/${serverName}.rpm ${install_main_dir}/init.d/${serverName}
+    ${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName}
+    #    ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord
+    ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord
+  fi
+
+  if ((${initd_mod} == 1)); then
+    ${csudo}chkconfig --add ${serverName} || :
+    ${csudo}chkconfig --level 2345 ${serverName} on || :
+    ${csudo}chkconfig --add tarbitratord || :
+    ${csudo}chkconfig --level 2345 tarbitratord on || :
+  elif ((${initd_mod} == 2)); then
+    ${csudo}insserv ${serverName} || :
+    ${csudo}insserv -d ${serverName} || :
+    ${csudo}insserv tarbitratord || :
+    ${csudo}insserv -d tarbitratord || :
+  elif ((${initd_mod} == 3)); then
+    ${csudo}update-rc.d ${serverName} defaults || :
+    ${csudo}update-rc.d tarbitratord defaults || :
+  fi
+}
+
+function clean_service_on_systemd() {
+  taosd_service_config="${service_config_dir}/${serverName}.service"
+  if systemctl is-active --quiet ${serverName}; then
+    echo "${productName} is running, stopping it..."
+    ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null
+  fi
+  ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null
   ${csudo}rm -f ${taosd_service_config}
 
   tarbitratord_service_config="${service_config_dir}/tarbitratord.service"
@@ -433,7 +652,7 @@ function clean_service_on_systemd() {
   if [ "$verMode" == "cluster" ]; then
     nginx_service_config="${service_config_dir}/nginxd.service"
     if systemctl is-active --quiet nginxd; then
-            echo "Nginx for TDengine is running, stopping it..."
+      echo "Nginx for ${productName} is running, stopping it..."
       ${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null
     fi
     ${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null
@@ -441,89 +660,277 @@ function clean_service_on_systemd() {
   fi
 }
 
-# taos:2345:respawn:/etc/init.d/taosd start
-
 function install_service_on_systemd() {
   clean_service_on_systemd
 
-    taosd_service_config="${service_config_dir}/taosd.service"
+  [ -f ${script_dir}/cfg/${serverName}.service ] &&
-    ${csudo} bash -c "echo '[Unit]'                             >> ${taosd_service_config}"
+    ${csudo}cp ${script_dir}/cfg/${serverName}.service \
-    ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}"
+      ${service_config_dir}/ || :
-    ${csudo} bash -c "echo 'After=network-online.target taosadapter.service'        >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'Wants=network-online.target taosadapter.service'        >> ${taosd_service_config}"
-    ${csudo} bash -c "echo                                      >> ${taosd_service_config}"
-    ${csudo} bash -c "echo '[Service]'                          >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'Type=simple'                        >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd'           >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh'         >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'TimeoutStopSec=1000000s'            >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'LimitNOFILE=infinity'               >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'LimitNPROC=infinity'                >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'LimitCORE=infinity'                 >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'TimeoutStartSec=0'                  >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'StandardOutput=null'                >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'Restart=always'                     >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'StartLimitBurst=3'                  >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'StartLimitInterval=60s'             >> ${taosd_service_config}"
-    #${csudo} bash -c "echo 'StartLimitIntervalSec=60s'          >> ${taosd_service_config}"
-    ${csudo} bash -c "echo                                      >> ${taosd_service_config}"
-    ${csudo} bash -c "echo '[Install]'                          >> ${taosd_service_config}"
-    ${csudo} bash -c "echo 'WantedBy=multi-user.target'         >> ${taosd_service_config}"
-    ${csudo} systemctl enable taosd
-
   ${csudo}systemctl daemon-reload
+
+  ${csudo}systemctl enable ${serverName}
+
+  [ -f ${script_dir}/cfg/tarbitratord.service ] &&
+    ${csudo}cp ${script_dir}/cfg/tarbitratord.service \
+      ${service_config_dir}/ || :
+  ${csudo}systemctl daemon-reload
+
+  if [ "$verMode" == "cluster" ]; then
+    [ -f ${script_dir}/cfg/nginxd.service ] &&
+      ${csudo}cp ${script_dir}/cfg/nginxd.service \
+        ${service_config_dir}/ || :
+    ${csudo}systemctl daemon-reload
+
+    if ! ${csudo}systemctl enable nginxd &>/dev/null; then
+      ${csudo}systemctl daemon-reexec
+      ${csudo}systemctl enable nginxd
+    fi
+    ${csudo}systemctl start nginxd
+  fi
+}
+
+function install_adapter_service() {
+  if ((${service_mod} == 0)); then
+    [ -f ${script_dir}/cfg/${adapterName}.service ] &&
+      ${csudo}cp ${script_dir}/cfg/${adapterName}.service \
+        ${service_config_dir}/ || :
+    ${csudo}systemctl daemon-reload
+  fi
 }
 
 function install_service() {
-    # if ((${service_mod}==0)); then
+  if ((${service_mod} == 0)); then
-    #     install_service_on_systemd
+    install_service_on_systemd
-    # elif ((${service_mod}==1)); then
+  elif ((${service_mod} == 1)); then
-    #     install_service_on_sysvinit
+    install_service_on_sysvinit
-    # else
+  else
-    #     # must manual stop taosd
+    kill_process ${serverName}
-        kill_process taosd
+  fi
-    # fi
 }
 
-function install_config() {
+vercomp() {
-    if [ ! -f ${cfg_install_dir}/${configFile} ]; then
+  if [[ $1 == $2 ]]; then
-        ${csudo}mkdir -p ${cfg_install_dir}
+    return 0
-        [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir}
+  fi
-        ${csudo}chmod 644 ${cfg_install_dir}/*
+  local IFS=.
+  local i ver1=($1) ver2=($2)
+  # fill empty fields in ver1 with zeros
+  for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do
+    ver1[i]=0
+  done
+
+  for ((i = 0; i < ${#ver1[@]}; i++)); do
+    if [[ -z ${ver2[i]} ]]; then
+      # fill empty fields in ver2 with zeros
+      ver2[i]=0
+    fi
+    if ((10#${ver1[i]} > 10#${ver2[i]})); then
+      return 1
+    fi
+    if ((10#${ver1[i]} < 10#${ver2[i]})); then
+      return 2
+    fi
+  done
+  return 0
+}
+
+function is_version_compatible() {
+
+  curr_version=$(ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}')
+
+  if [ -f ${script_dir}/driver/vercomp.txt ]; then
+    min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt)
+  else
+    min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5)
   fi
 
-    ${csudo}cp -f ${script_dir}/cfg/${configFile} ${install_main_dir}/cfg/${configFile}.org
+  exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3)
-    ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg
+  vercomp $exist_version "2.0.16.0"
+  case $? in
+  2)
+    prompt_force=1
+    ;;
+  esac
+
+  vercomp $curr_version $min_compatible_version
+  echo "" # avoid $? value not update
+
+  case $? in
+  0) return 0 ;;
+  1) return 0 ;;
+  2) return 1 ;;
+  esac
 }
 
-function install_TDengine() {
+function updateProduct() {
-    # Start to install
+  # Check if version compatible
-    echo -e "${GREEN}Start to install TDengine...${NC}"
+  if ! is_version_compatible; then
+    echo -e "${RED}Version incompatible${NC}"
+    return 1
+  fi
+
+  # Start to update
+  if [ ! -e ${tarName} ]; then
+    echo "File ${tarName} does not exist"
+    exit 1
+  fi
+  tar -zxf ${tarName}
+  install_jemalloc
+
+  echo -e "${GREEN}Start to update ${productName}...${NC}"
+  # Stop the service if running
+  if pidof ${serverName} &>/dev/null; then
+    if ((${service_mod} == 0)); then
+      ${csudo}systemctl stop ${serverName} || :
+    elif ((${service_mod} == 1)); then
+      ${csudo}service ${serverName} stop || :
+    else
+      kill_process ${serverName}
+    fi
+    sleep 1
+  fi
+
+  if [ "$verMode" == "cluster" ]; then
+    if pidof nginx &>/dev/null; then
+      if ((${service_mod} == 0)); then
+        ${csudo}systemctl stop nginxd || :
+      elif ((${service_mod} == 1)); then
+        ${csudo}service nginxd stop || :
+      else
+        kill_process nginx
+      fi
+      sleep 1
+    fi
+  fi
 
   install_main_path
-    install_data
+
   install_log
   install_header
   install_lib
-    install_taosTools
+
+  if [ "$verMode" == "cluster" ]; then
+      install_connector
+  fi
+
+  install_examples
+  if [ -z $1 ]; then
+    install_bin
+    install_service
+    install_adapter_service
+    install_config
+    install_adapter_config
+
+    openresty_work=false
+    if [ "$verMode" == "cluster" ]; then
+      # Check if openresty is installed
+      # Check if nginx is installed successfully
+      if type curl &>/dev/null; then
+        if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
+          echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}"
+          openresty_work=true
+        else
+          echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
+        fi
+      fi
+    fi
+
+    echo
+    echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
+    echo -e "${GREEN_DARK}To configure Adapter (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml"
+    if ((${service_mod} == 0)); then
+      echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}systemctl start ${serverName}${NC}"
+    elif ((${service_mod} == 1)); then
+      echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}service ${serverName} start${NC}"
+    else
+      echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}"
+      echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ./${serverName}${NC}"
+    fi
+
+    if [ ${openresty_work} = 'true' ]; then
+      echo -e "${GREEN_DARK}To access ${productName}    ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}"
+    else
+      echo -e "${GREEN_DARK}To access ${productName}    ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}"
+    fi
+
+    if ((${prompt_force} == 1)); then
+      echo ""
+      echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}"
+    fi
+    echo
+    echo -e "\033[44;32;1m${productName} is updated successfully!${NC}"
+  else
+    install_bin
+    install_config
+
+    echo
+    echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}"
+  fi
+
+  rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
+}
+
+function installProduct() {
+  # Start to install
+  if [ ! -e ${tarName} ]; then
+    echo "File ${tarName} does not exist"
+    exit 1
+  fi
+  tar -zxf ${tarName}
+
+  echo -e "${GREEN}Start to install ${productName}...${NC}"
+
+  install_main_path
+
+  if [ -z $1 ]; then
+    install_data
+  fi
+
+  install_log
+  install_header
+  install_lib
+  install_jemalloc
+  #install_avro lib
+  #install_avro lib64
+
+  if [ "$verMode" == "cluster" ]; then
+      install_connector
+  fi
+  install_examples
 
   if [ -z $1 ]; then # install service and client
     # For installing new
     install_bin
     install_service
+    install_adapter_service
+    install_adapter_config
+
+    openresty_work=false
+    if [ "$verMode" == "cluster" ]; then
+      # Check if nginx is installed successfully
+      if type curl &>/dev/null; then
+        if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then
+          echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}"
+          openresty_work=true
+        else
+          echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m"
+        fi
+      fi
+    fi
+
     install_config
 
     # Ask if to start the service
-        #echo
-        #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
     echo
-        echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg"
+    echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}"
+    echo -e "${GREEN_DARK}To configure ${adapterName} (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml"
     if ((${service_mod} == 0)); then
-          echo -e "${GREEN_DARK}To start TDengine     ${NC}: ${csudo} systemctl start taosd${NC}"
+      echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}systemctl start ${serverName}${NC}"
     elif ((${service_mod} == 1)); then
-          echo -e "${GREEN_DARK}To start TDengine     ${NC}: ${csudo} service taosd start${NC}"
+      echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${csudo}service ${serverName} start${NC}"
     else
-          echo -e "${GREEN_DARK}To start TDengine     ${NC}: taosd${NC}"
+      echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}"
+      echo -e "${GREEN_DARK}To start ${productName}     ${NC}: ${serverName}${NC}"
     fi
 
     if [ ! -z "$firstEp" ]; then
@@ -535,39 +942,49 @@ function install_TDengine() {
         tmpPort=""
       fi
       if [[ "$tmpPort" != "" ]]; then
-                echo -e "${GREEN_DARK}To access TDengine    ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
+        echo -e "${GREEN_DARK}To access ${productName}    ${NC}: ${clientName} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
       else
-                echo -e "${GREEN_DARK}To access TDengine    ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
+        echo -e "${GREEN_DARK}To access ${productName}    ${NC}: ${clientName} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
       fi
       echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
       echo
     elif [ ! -z "$serverFqdn" ]; then
-            echo -e "${GREEN_DARK}To access TDengine    ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
+      echo -e "${GREEN_DARK}To access ${productName}    ${NC}: ${clientName} -h $serverFqdn${GREEN_DARK} to login into ${productName} server${NC}"
       echo
     fi
 
-        echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
+    echo -e "\033[44;32;1m${productName} is installed successfully!${NC}"
     echo
   else # Only install client
     install_bin
     install_config
     echo
-        echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}"
+    echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}"
   fi
 
-    touch ~/.taos_history
+  touch ~/.${historyFile}
+  rm -rf $(tar -tf ${tarName} | grep -v "^\./$")
 }
 
-
 ## ==============================Main program starts from here============================
 serverFqdn=$(hostname)
 if [ "$verType" == "server" ]; then
   # Install server and client
-    install_TDengine
+  if [ -x ${bin_dir}/${serverName} ]; then
+    update_flag=1
+    updateProduct
+  else
+    installProduct
+  fi
 elif [ "$verType" == "client" ]; then
   interactiveFqdn=no
   # Only install client
-    install_TDengine client
+  if [ -x ${bin_dir}/${clientName} ]; then
+    update_flag=1
+    updateProduct client
+  else
+    installProduct client
+  fi
 else
   echo "please input correct verType"
 fi

source/common/src/tmsg.c

@@ -28,7 +28,7 @@
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
-int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
+int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
   if (pMsg == NULL) {
     terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
     return -1;
@@ -102,7 +102,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
   }
 }
 
-int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) {
+int32_t tPrintFixedSchemaSubmitReq(SSubmitReq *pReq, STSchema *pTschema) {
   SSubmitMsgIter msgIter = {0};
   if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1;
   while (true) {

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -113,6 +113,8 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
     SRpcMsg  rsp = {.info = pMsg->info};
 
+    vnodePreprocessReq(pVnode->pImpl, pMsg);
+
     int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false);
     if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
       dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId);

source/dnode/snode/src/snode.c

@@ -57,9 +57,7 @@ void sndMetaDelete(SStreamMeta *pMeta) {
 }
 
 int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) {
-  for (int i = 0; i < pTask->exec.numOfRunners; i++) {
+  pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
-    pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL);
-  }
   return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *));
 }
 

source/dnode/vnode/CMakeLists.txt

@@ -48,7 +48,6 @@ target_sources(
     # tq
     "src/tq/tq.c"
     "src/tq/tqCommit.c"
-    "src/tq/tqMetaStore.c"
     "src/tq/tqOffset.c"
     "src/tq/tqPush.c"
     "src/tq/tqRead.c"

source/dnode/vnode/inc/vnode.h

@@ -51,7 +51,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs);
 void    vnodeDestroy(const char *path, STfs *pTfs);
 SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
 void    vnodeClose(SVnode *pVnode);
-int32_t vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version);
+int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp);
 int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
 int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
@@ -177,7 +177,7 @@ struct SMetaEntry {
   int64_t  version;
   int8_t   type;
   tb_uid_t uid;
-  const char *name;
+  char    *name;
   union {
     struct {
       SSchemaWrapper schema;
@@ -187,7 +187,7 @@ struct SMetaEntry {
       int64_t  ctime;
       int32_t  ttlDays;
       tb_uid_t suid;
-      const uint8_t *pTags;
+      uint8_t *pTags;
     } ctbEntry;
     struct {
       int64_t        ctime;

source/dnode/vnode/src/inc/tq.h

@@ -41,45 +41,6 @@ extern "C" {
 #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ  ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
-#define TQ_BUFFER_SIZE 4
-
-#define TQ_BUCKET_MASK 0xFF
-#define TQ_BUCKET_SIZE 256
-
-#define TQ_PAGE_SIZE 4096
-// key + offset + size
-#define TQ_IDX_SIZE 24
-// 4096 / 24
-#define TQ_MAX_IDX_ONE_PAGE 170
-// 24 * 170
-#define TQ_IDX_PAGE_BODY_SIZE 4080
-// 4096 - 4080
-#define TQ_IDX_PAGE_HEAD_SIZE 16
-
-#define TQ_ACTION_CONST      0
-#define TQ_ACTION_INUSE      1
-#define TQ_ACTION_INUSE_CONT 2
-#define TQ_ACTION_INTXN      3
-
-#define TQ_SVER 0
-
-// TODO: inplace mode is not implemented
-#define TQ_UPDATE_INPLACE 0
-#define TQ_UPDATE_APPEND  1
-
-#define TQ_DUP_INTXN_REWRITE 0
-#define TQ_DUP_INTXN_REJECT  2
-
-static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
-
-static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
-
-static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
-
-#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
-
-typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus;
-
 typedef struct STqOffsetCfg   STqOffsetCfg;
 typedef struct STqOffsetStore STqOffsetStore;
 
@@ -98,53 +59,6 @@ struct STqReadHandle {
   STSchema*         pSchema;
 };
 
-typedef struct {
-  int16_t ver;
-  int16_t action;
-  int32_t checksum;
-  int64_t ssize;
-  char    content[];
-} STqSerializedHead;
-
-typedef int32_t (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
-typedef int32_t (*FTqDeserialize)(void* self, const STqSerializedHead* pHead, void** ppObj);
-typedef void (*FTqDelete)(void*);
-
-typedef struct {
-  int64_t key;
-  int64_t offset;
-  int64_t serializedSize;
-  void*   valueInUse;
-  void*   valueInTxn;
-} STqMetaHandle;
-
-typedef struct STqMetaList {
-  STqMetaHandle       handle;
-  struct STqMetaList* next;
-  // struct STqMetaList* inTxnPrev;
-  // struct STqMetaList* inTxnNext;
-  struct STqMetaList* unpersistPrev;
-  struct STqMetaList* unpersistNext;
-} STqMetaList;
-
-typedef struct {
-  STQ*         pTq;
-  STqMetaList* bucket[TQ_BUCKET_SIZE];
-  // a table head
-  STqMetaList* unpersistHead;
-  // topics that are not connectted
-  STqMetaList* unconnectTopic;
-
-  TdFilePtr pFile;
-  TdFilePtr pIdxFile;
-
-  char*          dirPath;
-  int32_t        tqConfigFlag;
-  FTqSerialize   pSerializer;
-  FTqDeserialize pDeserializer;
-  FTqDelete      pDeleter;
-} STqMetaStore;
-
 typedef struct {
   int64_t  consumerId;
   int32_t  epoch;
@@ -174,7 +88,6 @@ typedef struct {
 
 struct STQ {
   char*     path;
-  // STqMetaStore* tqMeta;
   SHashObj* pushMgr;  // consumerId -> STqExec*
   SHashObj* execs;    // subKey -> STqExec
   SHashObj* pStreamTasks;
@@ -190,87 +103,10 @@ typedef struct {
 
 static STqMgmt tqMgmt;
 
-typedef struct {
-  int8_t         status;
-  int64_t        offset;
-  qTaskInfo_t    task;
-  STqReadHandle* pReadHandle;
-} STqTaskItem;
-
-// new version
-typedef struct {
-  int64_t     firstOffset;
-  int64_t     lastOffset;
-  STqTaskItem output[TQ_BUFFER_SIZE];
-} STqBuffer;
-
-typedef struct {
-  char            topicName[TSDB_TOPIC_FNAME_LEN];
-  char*           sql;
-  char*           logicalPlan;
-  char*           physicalPlan;
-  char*           qmsg;
-  STqBuffer       buffer;
-  SWalReadHandle* pReadhandle;
-} STqTopic;
-
-typedef struct {
-  int64_t consumerId;
-  int32_t epoch;
-  char    cgroup[TSDB_TOPIC_FNAME_LEN];
-  SArray* topics;  // SArray<STqTopic>
-} STqConsumer;
-
-typedef struct {
-  int8_t      type;
-  int8_t      nodeType;
-  int8_t      reserved[6];
-  int64_t     streamId;
-  qTaskInfo_t task;
-  // TODO sync function
-} STqStreamPusher;
-
-typedef struct {
-  int8_t inited;
-  tmr_h  timer;
-} STqPushMgmt;
-
-static STqPushMgmt tqPushMgmt;
-
 // init once
 int  tqInit();
 void tqCleanUp();
 
-// open in each vnode
-// required by vnode
-
-int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
-int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);
-
-static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; }
-
-// tqMetaStore.h
-STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer,
-                          FTqDelete pDeleter, int32_t tqConfigFlag);
-int32_t       tqStoreClose(STqMetaStore*);
-// int32_t       tqStoreDelete(TqMetaStore*);
-// int32_t       tqStoreCommitAll(TqMetaStore*);
-int32_t tqStorePersist(STqMetaStore*);
-// clean deleted idx and data from persistent file
-int32_t tqStoreCompact(STqMetaStore*);
-
-void* tqHandleGet(STqMetaStore*, int64_t key);
-// make it unpersist
-void*   tqHandleTouchGet(STqMetaStore*, int64_t key);
-int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
-int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
-// delete committed kv pair
-// notice that a delete action still needs to be committed
-int32_t tqHandleDel(STqMetaStore*, int64_t key);
-int32_t tqHandlePurge(STqMetaStore*, int64_t key);
-int32_t tqHandleCommit(STqMetaStore*, int64_t key);
-int32_t tqHandleAbort(STqMetaStore*, int64_t key);
-
 // tqOffset
 STqOffsetStore* STqOffsetOpen(STqOffsetCfg*);
 void            STqOffsetClose(STqOffsetStore*);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -104,7 +104,7 @@ int          tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep
 int          tsdbClose(STsdb** pTsdb);
 int          tsdbBegin(STsdb* pTsdb);
 int          tsdbCommit(STsdb* pTsdb);
-int          tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, const SSubmitReq* pMsg);
+int          tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
 int          tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
 tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
@@ -123,11 +123,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
-#if 0
+int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data);
-int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
-int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
-#endif
-int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data);
 int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);

source/dnode/vnode/src/meta/metaTable.c

@@ -165,7 +165,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
   ASSERT(ret == 0);
 
-  tDecoderInit(&dc, pData, nData);
+  oStbEntry.pBuf = taosMemoryMalloc(nData);
+  memcpy(oStbEntry.pBuf, pData, nData);
+  tDecoderInit(&dc, oStbEntry.pBuf, nData);
   metaDecodeEntry(&dc, &oStbEntry);
 
   nStbEntry.version = version;
@@ -193,6 +195,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   // update uid index
   tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0);
 
+  if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
   metaULock(pMeta);
   tDecoderClear(&dc);
   tdbTbcClose(pTbDbc);
@@ -220,9 +223,6 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
     terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
     metaReaderClear(&mr);
     return -1;
-  } else {
-    pReq->uid = tGenIdPI64();
-    pReq->ctime = taosGetTimestampMs();
   }
   metaReaderClear(&mr);
 
@@ -420,7 +420,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
 
   // get table entry
   SDecoder dc = {0};
-  tDecoderInit(&dc, pData, nData);
+  entry.pBuf = taosMemoryMalloc(nData);
+  memcpy(entry.pBuf, pData, nData);
+  tDecoderInit(&dc, entry.pBuf, nData);
   ret = metaDecodeEntry(&dc, &entry);
   ASSERT(ret == 0);
 

source/dnode/vnode/src/tq/tq.c

@@ -36,15 +36,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
   /*ASSERT(0);*/
   /*}*/
 
-#if 0
-  pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
-                            (FTqDelete)taosMemoryFree, 0);
-  if (pTq->tqMeta == NULL) {
-    taosMemoryFree(pTq);
-    return NULL;
-  }
-#endif
-
   pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
 
   pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@@ -65,48 +56,6 @@ void tqClose(STQ* pTq) {
   // TODO
 }
 
-static void tdSRowDemo() {
-#define DEMO_N_COLS 3
-
-  int16_t     schemaVersion = 0;
-  int32_t     numOfCols = DEMO_N_COLS;  // ts + int
-  SRowBuilder rb = {0};
-
-  SSchema schema[DEMO_N_COLS] = {
-      {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = COL_SMA_ON},
-      {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = COL_SMA_ON},
-      {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = COL_SMA_ON}};
-
-  SSchema*  pSchema = schema;
-  STSchema* pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols);
-
-  tdSRowInit(&rb, schemaVersion);
-  tdSRowSetTpInfo(&rb, numOfCols, pTSChema->flen);
-  int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSChema);
-  void*   row = taosMemoryCalloc(1, maxLen);  // make sure the buffer is enough
-
-  // set row buf
-  tdSRowResetBuf(&rb, row);
-
-  for (int32_t idx = 0; idx < pTSChema->numOfCols; ++idx) {
-    STColumn* pColumn = pTSChema->columns + idx;
-    if (idx == 0) {
-      int64_t tsKey = 1651234567;
-      tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &tsKey, true, pColumn->offset, idx);
-    } else if (idx == 1) {
-      int32_t val1 = 10;
-      tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &val1, true, pColumn->offset, idx);
-    } else {
-      tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, true, pColumn->offset, idx);
-    }
-  }
-
-  // print
-  tdSRowPrint(row, pTSChema, __func__);
-
-  taosMemoryFree(pTSChema);
-}
-
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
   void* pIter = NULL;
   while (1) {
@@ -261,166 +210,26 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
 }
 
 int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
-  if (msgType != TDMT_VND_SUBMIT) return 0;
+  if (msgType == TDMT_VND_SUBMIT) {
-
-  // make sure msgType == TDMT_VND_SUBMIT
-  if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
-    return -1;
-  }
-
     if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0;
 
+    if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
+      // TODO handle sma error
+    }
     void* data = taosMemoryMalloc(msgLen);
     if (data == NULL) {
       return -1;
     }
     memcpy(data, msg, msgLen);
 
-  tqProcessStreamTriggerNew(pTq, data);
+    tqProcessStreamTrigger(pTq, data);
-
+  }
-#if 0
-  SRpcMsg req = {
-      .msgType = TDMT_VND_STREAM_TRIGGER,
-      .pCont = data,
-      .contLen = msgLen,
-  };
-
-  tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
-#endif
 
   return 0;
 }
 
 int tqCommit(STQ* pTq) {
   // do nothing
-  /*return tqStorePersist(pTq->tqMeta);*/
-  return 0;
-}
-
-int32_t tqGetTopicHandleSize(const STqTopic* pTopic) {
-  return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) +
-         sizeof(int64_t) * 3;
-}
-
-int32_t tqGetConsumerHandleSize(const STqConsumer* pConsumer) {
-  int     num = taosArrayGetSize(pConsumer->topics);
-  int32_t sz = 0;
-  for (int i = 0; i < num; i++) {
-    STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
-    sz += tqGetTopicHandleSize(pTopic);
-  }
-  return sz;
-}
-
-static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) {
-  int32_t tlen = 0;
-  tlen += taosEncodeString(buf, pTopic->topicName);
-  /*tlen += taosEncodeString(buf, pTopic->sql);*/
-  /*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/
-  tlen += taosEncodeString(buf, pTopic->qmsg);
-  /*tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);*/
-  /*tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);*/
-  /*tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);*/
-  return tlen;
-}
-
-static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopic) {
-  buf = taosDecodeStringTo(buf, pTopic->topicName);
-  /*buf = taosDecodeString(buf, &pTopic->sql);*/
-  /*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/
-  buf = taosDecodeString(buf, &pTopic->qmsg);
-  /*buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);*/
-  /*buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);*/
-  /*buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);*/
-  return buf;
-}
-
-static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pConsumer) {
-  int32_t sz;
-
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
-  tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
-  tlen += taosEncodeString(buf, pConsumer->cgroup);
-  sz = taosArrayGetSize(pConsumer->topics);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
-    tlen += tEncodeSTqTopic(buf, pTopic);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer* pConsumer) {
-  int32_t sz;
-
-  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
-  buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
-  buf = taosDecodeStringTo(buf, pConsumer->cgroup);
-  buf = taosDecodeFixedI32(buf, &sz);
-  pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic));
-  if (pConsumer->topics == NULL) return NULL;
-  for (int32_t i = 0; i < sz; i++) {
-    STqTopic pTopic;
-    buf = tDecodeSTqTopic(buf, &pTopic);
-    taosArrayPush(pConsumer->topics, &pTopic);
-  }
-  return buf;
-}
-
-int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead) {
-  int32_t sz = tEncodeSTqConsumer(NULL, pConsumer);
-
-  if (sz > (*ppHead)->ssize) {
-    void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz);
-    if (tmpPtr == NULL) {
-      taosMemoryFree(*ppHead);
-      terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-      return -1;
-    }
-    *ppHead = tmpPtr;
-    (*ppHead)->ssize = sz;
-  }
-
-  void* ptr = (*ppHead)->content;
-  void* abuf = ptr;
-  tEncodeSTqConsumer(&abuf, pConsumer);
-
-  return 0;
-}
-
-int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) {
-  const void* str = pHead->content;
-  *ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer));
-  if (*ppConsumer == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return -1;
-  }
-  if (tDecodeSTqConsumer(str, *ppConsumer) == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return -1;
-  }
-  STqConsumer* pConsumer = *ppConsumer;
-  int32_t      sz = taosArrayGetSize(pConsumer->topics);
-  for (int32_t i = 0; i < sz; i++) {
-    STqTopic* pTopic = taosArrayGet(pConsumer->topics, i);
-    pTopic->pReadhandle = walOpenReadHandle(pTq->pWal);
-    if (pTopic->pReadhandle == NULL) {
-      ASSERT(false);
-    }
-    for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
-      pTopic->buffer.output[j].status = 0;
-      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
-      SReadHandle    handle = {
-             .reader = pReadHandle,
-             .meta = pTq->pVnode->pMeta,
-             .pMsgCb = &pTq->pVnode->msgCb,
-      };
-      pTopic->buffer.output[j].pReadHandle = pReadHandle;
-      pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
-    }
-  }
-
   return 0;
 }
 
@@ -685,213 +494,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   return 0;
 }
 
-#if 0
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReq* pReq = pMsg->pCont;
-  int64_t     consumerId = pReq->consumerId;
-  int64_t     fetchOffset;
-  int64_t     blockingTime = pReq->blockingTime;
-  int32_t     reqEpoch = pReq->epoch;
-
-  if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
-    fetchOffset = walGetFirstVer(pTq->pWal);
-  } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
-    fetchOffset = walGetLastVer(pTq->pWal);
-  } else {
-    fetchOffset = pReq->currentOffset + 1;
-  }
-
-  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
-         TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
-
-  SMqPollRspV2 rspV2 = {0};
-  rspV2.dataLen = 0;
-
-  STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
-  if (pConsumer == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch);
-  while (consumerEpoch < reqEpoch) {
-    consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch);
-  }
-
-  STqTopic* pTopic = NULL;
-  int32_t   topicSz = taosArrayGetSize(pConsumer->topics);
-  for (int32_t i = 0; i < topicSz; i++) {
-    STqTopic* topic = taosArrayGet(pConsumer->topics, i);
-    // TODO race condition
-    ASSERT(pConsumer->consumerId == consumerId);
-    if (strcmp(topic->topicName, pReq->topic) == 0) {
-      pTopic = topic;
-      break;
-    }
-  }
-  if (pTopic == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
-          TD_VID(pTq->pVnode));
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = -1;
-    tmsgSendRsp(pMsg);
-    return 0;
-  }
-
-  tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
-         TD_VID(pTq->pVnode));
-
-  rspV2.reqOffset = pReq->currentOffset;
-  rspV2.skipLogNum = 0;
-
-  while (1) {
-    /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
-    // TODO
-    consumerEpoch = atomic_load_32(&pConsumer->epoch);
-    if (consumerEpoch > reqEpoch) {
-      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
-             consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
-      break;
-    }
-    SWalReadHead* pHead;
-    if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
-      // TODO: no more log, set timer to wait blocking time
-      // if data inserted during waiting, launch query and
-      // response to user
-      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
-             TD_VID(pTq->pVnode), fetchOffset);
-      break;
-    }
-    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-           TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
-    /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
-    /*pHead = pTopic->pReadhandle->pHead;*/
-    if (pHead->msgType == TDMT_VND_SUBMIT) {
-      SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-      qTaskInfo_t task = pTopic->buffer.output[workerId].task;
-      ASSERT(task);
-      qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
-      SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
-      while (1) {
-        SSDataBlock* pDataBlock = NULL;
-        uint64_t     ts;
-        if (qExecTask(task, &pDataBlock, &ts) < 0) {
-          ASSERT(false);
-        }
-        if (pDataBlock == NULL) {
-          /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
-          break;
-        }
-
-        taosArrayPush(pRes, pDataBlock);
-      }
-
-      if (taosArrayGetSize(pRes) == 0) {
-        tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
-               pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
-        fetchOffset++;
-        rspV2.skipLogNum++;
-        taosArrayDestroy(pRes);
-        continue;
-      }
-      rspV2.rspOffset = fetchOffset;
-
-      int32_t blockSz = taosArrayGetSize(pRes);
-      int32_t dataBlockStrLen = 0;
-      for (int32_t i = 0; i < blockSz; i++) {
-        SSDataBlock* pBlock = taosArrayGet(pRes, i);
-        dataBlockStrLen += sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
-      }
-
-      void* dataBlockBuf = taosMemoryMalloc(dataBlockStrLen);
-      if (dataBlockBuf == NULL) {
-        pMsg->code = -1;
-        taosMemoryFree(pHead);
-      }
-
-      rspV2.blockData = dataBlockBuf;
-
-      int32_t pos;
-      rspV2.blockPos = taosArrayInit(blockSz, sizeof(int32_t));
-      for (int32_t i = 0; i < blockSz; i++) {
-        pos = 0;
-        SSDataBlock*       pBlock = taosArrayGet(pRes, i);
-        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)dataBlockBuf;
-        pRetrieve->useconds = 0;
-        pRetrieve->precision = 0;
-        pRetrieve->compressed = 0;
-        pRetrieve->completed = 1;
-        pRetrieve->numOfRows = htonl(pBlock->info.rows);
-        blockCompressEncode(pBlock, pRetrieve->data, &pos, pBlock->info.numOfCols, false);
-        taosArrayPush(rspV2.blockPos, &rspV2.dataLen);
-
-        int32_t totLen = sizeof(SRetrieveTableRsp) + pos;
-        pRetrieve->compLen = htonl(totLen);
-        rspV2.dataLen += totLen;
-        dataBlockBuf = POINTER_SHIFT(dataBlockBuf, totLen);
-      }
-      ASSERT(POINTER_DISTANCE(dataBlockBuf, rspV2.blockData) <= dataBlockStrLen);
-
-      int32_t msgLen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2);
-      void*   buf = rpcMallocCont(msgLen);
-
-      ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
-      ((SMqRspHead*)buf)->epoch = pReq->epoch;
-      ((SMqRspHead*)buf)->consumerId = consumerId;
-
-      void* msgBodyBuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-      tEncodeSMqPollRspV2(&msgBodyBuf, &rspV2);
-
-      /*rsp.pBlockData = pRes;*/
-
-      /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/
-      SRpcMsg resp = {.info = pMsg->info, pCont = buf, .contLen = msgLen, .code = 0};
-      tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
-             pHead->msgType, consumerId, pReq->epoch);
-      tmsgSendRsp(&resp);
-      taosMemoryFree(pHead);
-      return 0;
-    } else {
-      taosMemoryFree(pHead);
-      fetchOffset++;
-      rspV2.skipLogNum++;
-    }
-  }
-
-  /*if (blockingTime != 0) {*/
-  /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
-  /*} else {*/
-
-  rspV2.rspOffset = fetchOffset - 1;
-
-  int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2);
-  void*   buf = rpcMallocCont(tlen);
-  if (buf == NULL) {
-    pMsg->code = -1;
-    return -1;
-  }
-  ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
-  ((SMqRspHead*)buf)->epoch = pReq->epoch;
-  ((SMqRspHead*)buf)->consumerId = consumerId;
-
-  void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
-  tEncodeSMqPollRspV2(&abuf, &rspV2);
-
-  SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
-  tmsgSendRsp(&resp);
-  tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
-         pReq->epoch);
-  /*}*/
-
-  return 0;
-}
-#endif
-
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
   SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
 
@@ -981,55 +583,6 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
   ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
 }
 
-int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
-  pTask->status = TASK_STATUS__IDLE;
-  pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
-  pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
-
-  pTask->inputQ = taosOpenQueue();
-  pTask->outputQ = taosOpenQueue();
-  pTask->inputQAll = taosAllocateQall();
-  pTask->outputQAll = taosAllocateQall();
-
-  if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
-    goto FAIL;
-
-  if (pTask->execType != TASK_EXEC__NONE) {
-    // expand runners
-    pTask->exec.numOfRunners = parallel;
-    pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner));
-    if (pTask->exec.runners == NULL) {
-      goto FAIL;
-    }
-    for (int32_t i = 0; i < parallel; i++) {
-      STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
-      SReadHandle    handle = {
-             .reader = pStreamReader,
-             .meta = pTq->pVnode->pMeta,
-             .pMsgCb = &pTq->pVnode->msgCb,
-             .vnode = pTq->pVnode,
-      };
-      pTask->exec.runners[i].inputHandle = pStreamReader;
-      pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
-      ASSERT(pTask->exec.runners[i].executor);
-    }
-  }
-
-  if (pTask->sinkType == TASK_SINK__TABLE) {
-    pTask->tbSink.vnode = pTq->pVnode;
-    pTask->tbSink.tbSinkFunc = tqTableSink;
-  }
-
-  return 0;
-FAIL:
-  if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
-  if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
-  if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
-  if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
-  if (pTask) taosMemoryFree(pTask);
-  return -1;
-}
-
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
   if (pTask == NULL) {
@@ -1042,9 +595,31 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   }
   tDecoderClear(&decoder);
 
+  pTask->status = TASK_STATUS__IDLE;
+  pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
+  pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
+
+  pTask->inputQ = taosOpenQueue();
+  pTask->outputQ = taosOpenQueue();
+  pTask->inputQAll = taosAllocateQall();
+  pTask->outputQAll = taosAllocateQall();
+
+  if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
+    goto FAIL;
+
   // exec
-  if (tqExpandTask(pTq, pTask, 4) < 0) {
+  if (pTask->execType != TASK_EXEC__NONE) {
-    ASSERT(0);
+    // expand runners
+    STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
+    SReadHandle    handle = {
+           .reader = pStreamReader,
+           .meta = pTq->pVnode->pMeta,
+           .pMsgCb = &pTq->pVnode->msgCb,
+           .vnode = pTq->pVnode,
+    };
+    pTask->exec.inputHandle = pStreamReader;
+    pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
+    ASSERT(pTask->exec.executor);
   }
 
   // sink
@@ -1052,8 +627,12 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   if (pTask->sinkType == TASK_SINK__SMA) {
     pTask->smaSink.smaSink = smaHandleRes;
   } else if (pTask->sinkType == TASK_SINK__TABLE) {
+    pTask->tbSink.vnode = pTq->pVnode;
+    pTask->tbSink.tbSinkFunc = tqTableSink;
+
     ASSERT(pTask->tbSink.pSchemaWrapper);
     ASSERT(pTask->tbSink.pSchemaWrapper->pSchema);
+
     pTask->tbSink.pTSchema =
         tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols);
     ASSERT(pTask->tbSink.pTSchema);
@@ -1061,94 +640,17 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
 
   taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask));
 
-  return 0;
-}
-
-int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId) {
-  void* pIter = NULL;
-
-  while (1) {
-    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
-    if (pIter == NULL) break;
-    SStreamTask* pTask = (SStreamTask*)pIter;
-
-    if (streamExecTask(pTask, &pTq->pVnode->msgCb, data, STREAM_DATA_TYPE_SUBMIT_BLOCK, workerId) < 0) {
-      // TODO
-    }
-  }
-  return 0;
-}
-
-#if 0
-int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data) {
-  SStreamDataSubmit* pSubmit = NULL;
-
-  // build data
-  pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
-  if (pSubmit == NULL) return -1;
-  pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t));
-  if (pSubmit->dataRef == NULL) goto FAIL;
-  *pSubmit->dataRef = 1;
-  pSubmit->data = data;
-  pSubmit->type = STREAM_INPUT__DATA_BLOCK;
-
-  void* pIter = NULL;
-  while (1) {
-    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
-    if (pIter == NULL) break;
-    SStreamTask* pTask = (SStreamTask*)pIter;
-    if (pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK) {
-      streamEnqueueDataSubmit(pTask, pSubmit);
-      // TODO cal back pressure
-    }
-    // check run
-    int8_t execStatus = atomic_load_8(&pTask->status);
-    if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
-      SStreamTaskRunReq* pReq = taosMemoryMalloc(sizeof(SStreamTaskRunReq));
-      if (pReq == NULL) continue;
-      // TODO: do we need htonl?
-      pReq->head.vgId = pTq->pVnode->config.vgId;
-      pReq->streamId = pTask->streamId;
-      pReq->taskId = pTask->taskId;
-      SRpcMsg msg = {
-          .msgType = 0,
-          .pCont = pReq,
-          .contLen = sizeof(SStreamTaskRunReq),
-      };
-      tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &msg);
-    }
-  }
-  streamDataSubmitRefDec(pSubmit);
-
   return 0;
 FAIL:
-  if (pSubmit) {
+  if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
-    if (pSubmit->dataRef) {
+  if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
-      taosMemoryFree(pSubmit->dataRef);
+  if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
-    }
+  if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
-    taosFreeQitem(pSubmit);
+  if (pTask) taosMemoryFree(pTask);
-  }
   return -1;
 }
-#endif
 
-int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) {
+int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
-  SStreamTaskExecReq req;
-  tDecodeSStreamTaskExecReq(msg, &req);
-
-  int32_t taskId = req.taskId;
-  ASSERT(taskId);
-
-  SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
-  ASSERT(pTask);
-
-  if (streamExecTask(pTask, &pTq->pVnode->msgCb, req.data, STREAM_DATA_TYPE_SSDATA_BLOCK, workerId) < 0) {
-    // TODO
-  }
-  return 0;
-}
-
-int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* pReq) {
   void* pIter = NULL;
   bool  failed = false;
 
@@ -1234,7 +736,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) {
   SStreamDispatchReq* pReq = pMsg->pCont;
   int32_t             taskId = pReq->taskId;
   SStreamTask*        pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
-  streamTaskProcessDispatchReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg);
+  streamProcessDispatchReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg);
   return 0;
 }
 
@@ -1242,7 +744,7 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) {
   SStreamTaskRecoverReq* pReq = pMsg->pCont;
   int32_t                taskId = pReq->taskId;
   SStreamTask*           pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
-  streamTaskProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg);
+  streamProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg);
   return 0;
 }
 
@@ -1250,7 +752,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
   SStreamDispatchRsp* pRsp = pMsg->pCont;
   int32_t             taskId = pRsp->taskId;
   SStreamTask*        pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
-  streamTaskProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp);
+  streamProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp);
   return 0;
 }
 
@@ -1258,6 +760,6 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) {
   SStreamTaskRecoverRsp* pRsp = pMsg->pCont;
   int32_t                taskId = pRsp->taskId;
   SStreamTask*           pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
-  streamTaskProcessRecoverRsp(pTask, pRsp);
+  streamProcessRecoverRsp(pTask, pRsp);
   return 0;
 }

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -1,622 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#include "tq.h"
-// #include <fcntl.h>
-// #include <string.h>
-// #include <unistd.h>
-// #include "osDir.h"
-
-#define TQ_META_NAME "tq.meta"
-#define TQ_IDX_NAME  "tq.idx"
-
-static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value);
-static void*   tqHandleGetUncommitted(STqMetaStore*, int64_t key);
-
-static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
-  if (pNode->unpersistNext == NULL) {
-    pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
-    pNode->unpersistPrev = pMeta->unpersistHead;
-    pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
-    pMeta->unpersistHead->unpersistNext = pNode;
-  }
-}
-
-static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
-  int offset = taosLSeekFile(pFile, 0, SEEK_END);
-  int pageNo = offset / TQ_PAGE_SIZE;
-  int curPageOffset = pageNo * TQ_PAGE_SIZE;
-  return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
-}
-
-// TODO: the struct is tightly coupled with index entry
-typedef struct STqIdxPageHead {
-  int16_t writeOffset;
-  int8_t  unused[14];
-} STqIdxPageHead;
-
-typedef struct STqIdxPageBuf {
-  STqIdxPageHead head;
-  char           buffer[TQ_IDX_PAGE_BODY_SIZE];
-} STqIdxPageBuf;
-
-static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
-  int offset = tqSeekLastPage(pFile);
-  int nBytes;
-  if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-  if (nBytes == 0) {
-    memset(pBuf, 0, TQ_PAGE_SIZE);
-    pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
-  }
-  ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
-
-  return taosLSeekFile(pFile, offset, SEEK_SET);
-}
-
-STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
-                          FTqDelete deleter, int32_t tqConfigFlag) {
-  STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore));
-  if (pMeta == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return NULL;
-  }
-  pMeta->pTq = pTq;
-
-  // concat data file name and index file name
-  size_t pathLen = strlen(path);
-  pMeta->dirPath = taosMemoryMalloc(pathLen + 1);
-  if (pMeta->dirPath == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    taosMemoryFree(pMeta);
-    return NULL;
-  }
-  strcpy(pMeta->dirPath, path);
-
-  char* name = taosMemoryMalloc(pathLen + 10);
-
-  strcpy(name, path);
-  if (!taosDirExist(name) && taosMkDir(name) != 0) {
-    terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR;
-    tqError("failed to create dir:%s since %s ", name, terrstr());
-  }
-  strcat(name, "/" TQ_IDX_NAME);
-  TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
-  if (pIdxFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    tqError("failed to open file:%s since %s ", name, terrstr());
-    // free memory
-    taosMemoryFree(name);
-    return NULL;
-  }
-
-  pMeta->pIdxFile = pIdxFile;
-  pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList));
-  if (pMeta->unpersistHead == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    taosMemoryFree(name);
-    return NULL;
-  }
-  pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
-
-  strcpy(name, path);
-  strcat(name, "/" TQ_META_NAME);
-  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
-  if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    tqError("failed to open file:%s since %s", name, terrstr());
-    taosMemoryFree(name);
-    return NULL;
-  }
-  taosMemoryFree(name);
-
-  pMeta->pFile = pFile;
-
-  pMeta->pSerializer = serializer;
-  pMeta->pDeserializer = deserializer;
-  pMeta->pDeleter = deleter;
-  pMeta->tqConfigFlag = tqConfigFlag;
-
-  // read idx file and load into memory
-  STqIdxPageBuf      idxBuf;
-  STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE);
-  if (serializedObj == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-  }
-  int  idxRead;
-  int  allocated = TQ_PAGE_SIZE;
-  bool readEnd = false;
-  while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
-    if (idxRead == -1) {
-      // TODO: handle error
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      tqError("failed to read tq index file since %s", terrstr());
-    }
-    ASSERT(idxBuf.head.writeOffset == idxRead);
-    // loop read every entry
-    for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
-      STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList));
-      if (pNode == NULL) {
-        terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-        // TODO: free memory
-      }
-      memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
-
-      taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
-      if (allocated < pNode->handle.serializedSize) {
-        void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize);
-        if (ptr == NULL) {
-          terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-          // TODO: free memory
-        }
-        serializedObj = ptr;
-        allocated = pNode->handle.serializedSize;
-      }
-      serializedObj->ssize = pNode->handle.serializedSize;
-      if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
-        // TODO: read error
-      }
-      if (serializedObj->action == TQ_ACTION_INUSE) {
-        if (serializedObj->ssize != sizeof(STqSerializedHead)) {
-          pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse);
-        } else {
-          pNode->handle.valueInUse = TQ_DELETE_TOKEN;
-        }
-      } else if (serializedObj->action == TQ_ACTION_INTXN) {
-        if (serializedObj->ssize != sizeof(STqSerializedHead)) {
-          pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInTxn);
-        } else {
-          pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
-        }
-      } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) {
-        if (serializedObj->ssize != sizeof(STqSerializedHead)) {
-          pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse);
-        } else {
-          pNode->handle.valueInUse = TQ_DELETE_TOKEN;
-        }
-        STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
-        if (ptr->ssize != sizeof(STqSerializedHead)) {
-          pMeta->pDeserializer(pTq, ptr, &pNode->handle.valueInTxn);
-        } else {
-          pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
-        }
-      } else {
-        ASSERT(0);
-      }
-
-      // put into list
-      int          bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
-      STqMetaList* pBucketNode = pMeta->bucket[bucketKey];
-      if (pBucketNode == NULL) {
-        pMeta->bucket[bucketKey] = pNode;
-      } else if (pBucketNode->handle.key == pNode->handle.key) {
-        pNode->next = pBucketNode->next;
-        pMeta->bucket[bucketKey] = pNode;
-      } else {
-        while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) {
-          pBucketNode = pBucketNode->next;
-        }
-        if (pBucketNode->next) {
-          ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
-          STqMetaList* pNodeFound = pBucketNode->next;
-          pNode->next = pNodeFound->next;
-          pBucketNode->next = pNode;
-          pBucketNode = pNodeFound;
-        } else {
-          pNode->next = pMeta->bucket[bucketKey];
-          pMeta->bucket[bucketKey] = pNode;
-          pBucketNode = NULL;
-        }
-      }
-      if (pBucketNode) {
-        if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-          pMeta->pDeleter(pBucketNode->handle.valueInUse);
-        }
-        if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-          pMeta->pDeleter(pBucketNode->handle.valueInTxn);
-        }
-        taosMemoryFree(pBucketNode);
-      }
-    }
-  }
-  taosMemoryFree(serializedObj);
-  return pMeta;
-}
-
-int32_t tqStoreClose(STqMetaStore* pMeta) {
-  // commit data and idx
-  tqStorePersist(pMeta);
-  ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
-  taosCloseFile(&pMeta->pFile);
-  taosCloseFile(&pMeta->pIdxFile);
-  // free memory
-  for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
-    STqMetaList* pNode = pMeta->bucket[i];
-    while (pNode) {
-      ASSERT(pNode->unpersistNext == NULL);
-      ASSERT(pNode->unpersistPrev == NULL);
-      if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-        pMeta->pDeleter(pNode->handle.valueInTxn);
-      }
-      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-        pMeta->pDeleter(pNode->handle.valueInUse);
-      }
-      STqMetaList* next = pNode->next;
-      taosMemoryFree(pNode);
-      pNode = next;
-    }
-  }
-  taosMemoryFree(pMeta->dirPath);
-  taosMemoryFree(pMeta->unpersistHead);
-  taosMemoryFree(pMeta);
-  return 0;
-}
-
-int32_t tqStoreDelete(STqMetaStore* pMeta) {
-  taosCloseFile(&pMeta->pFile);
-  taosCloseFile(&pMeta->pIdxFile);
-  // free memory
-  for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
-    STqMetaList* pNode = pMeta->bucket[i];
-    pMeta->bucket[i] = NULL;
-    while (pNode) {
-      if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-        pMeta->pDeleter(pNode->handle.valueInTxn);
-      }
-      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-        pMeta->pDeleter(pNode->handle.valueInUse);
-      }
-      STqMetaList* next = pNode->next;
-      taosMemoryFree(pNode);
-      pNode = next;
-    }
-  }
-  taosMemoryFree(pMeta->unpersistHead);
-  taosRemoveDir(pMeta->dirPath);
-  taosMemoryFree(pMeta->dirPath);
-  taosMemoryFree(pMeta);
-  return 0;
-}
-
-int32_t tqStorePersist(STqMetaStore* pMeta) {
-  STqIdxPageBuf      idxBuf;
-  int64_t*           bufPtr = (int64_t*)idxBuf.buffer;
-  STqMetaList*       pHead = pMeta->unpersistHead;
-  STqMetaList*       pNode = pHead->unpersistNext;
-  STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead));
-  if (pSHead == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return -1;
-  }
-  pSHead->ver = TQ_SVER;
-  pSHead->checksum = 0;
-  pSHead->ssize = sizeof(STqSerializedHead);
-  /*int allocatedSize = sizeof(STqSerializedHead);*/
-  int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
-
-  tqReadLastPage(pMeta->pIdxFile, &idxBuf);
-
-  if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
-    taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
-    memset(&idxBuf, 0, TQ_PAGE_SIZE);
-    idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
-  } else {
-    bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
-  }
-
-  while (pHead != pNode) {
-    int nBytes = 0;
-
-    if (pNode->handle.valueInUse) {
-      if (pNode->handle.valueInTxn) {
-        pSHead->action = TQ_ACTION_INUSE_CONT;
-      } else {
-        pSHead->action = TQ_ACTION_INUSE;
-      }
-
-      if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
-        pSHead->ssize = sizeof(STqSerializedHead);
-      } else {
-        pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
-      }
-      nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
-      ASSERT(nBytes == pSHead->ssize);
-    }
-
-    if (pNode->handle.valueInTxn) {
-      pSHead->action = TQ_ACTION_INTXN;
-      if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
-        pSHead->ssize = sizeof(STqSerializedHead);
-      } else {
-        pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
-      }
-      int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
-      ASSERT(nBytesTxn == pSHead->ssize);
-      nBytes += nBytesTxn;
-    }
-    pNode->handle.offset = offset;
-    offset += nBytes;
-
-    // write idx file
-    // TODO: endian check and convert
-    *(bufPtr++) = pNode->handle.key;
-    *(bufPtr++) = pNode->handle.offset;
-    *(bufPtr++) = (int64_t)nBytes;
-    idxBuf.head.writeOffset += TQ_IDX_SIZE;
-
-    if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
-      nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
-      // TODO: handle error with tfile
-      ASSERT(nBytes == TQ_PAGE_SIZE);
-      memset(&idxBuf, 0, TQ_PAGE_SIZE);
-      idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
-      bufPtr = (int64_t*)&idxBuf.buffer;
-    }
-    // remove from unpersist list
-    pHead->unpersistNext = pNode->unpersistNext;
-    pHead->unpersistNext->unpersistPrev = pHead;
-    pNode->unpersistPrev = pNode->unpersistNext = NULL;
-    pNode = pHead->unpersistNext;
-
-    // remove from bucket
-    if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) {
-      int          bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
-      STqMetaList* pBucketHead = pMeta->bucket[bucketKey];
-      if (pBucketHead == pNode) {
-        pMeta->bucket[bucketKey] = pNode->next;
-      } else {
-        STqMetaList* pBucketNode = pBucketHead;
-        while (pBucketNode->next != NULL && pBucketNode->next != pNode) {
-          pBucketNode = pBucketNode->next;
-        }
-        // impossible for pBucket->next == NULL
-        ASSERT(pBucketNode->next == pNode);
-        pBucketNode->next = pNode->next;
-      }
-      taosMemoryFree(pNode);
-    }
-  }
-
-  // write left bytes
-  taosMemoryFree(pSHead);
-  // TODO: write new version in tfile
-  if ((char*)bufPtr != idxBuf.buffer) {
-    int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
-    // TODO: handle error in tfile
-    ASSERT(nBytes == idxBuf.head.writeOffset);
-  }
-  // TODO: using fsync in tfile
-  taosFsyncFile(pMeta->pIdxFile);
-  taosFsyncFile(pMeta->pFile);
-  return 0;
-}
-
-static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-        pMeta->pDeleter(pNode->handle.valueInUse);
-      }
-      // change pointer ownership
-      pNode->handle.valueInUse = value;
-      return 0;
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList));
-  if (pNewNode == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return -1;
-  }
-  pNewNode->handle.key = key;
-  pNewNode->handle.valueInUse = value;
-  pNewNode->next = pMeta->bucket[bucketKey];
-  // put into unpersist list
-  pNewNode->unpersistPrev = pMeta->unpersistHead;
-  pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
-  pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode;
-  pMeta->unpersistHead->unpersistNext = pNewNode;
-  return 0;
-}
-
-void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-        return pNode->handle.valueInUse;
-      } else {
-        return NULL;
-      }
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  return NULL;
-}
-
-void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-        tqLinkUnpersist(pMeta, pNode);
-        return pNode->handle.valueInUse;
-      } else {
-        return NULL;
-      }
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  return NULL;
-}
-
-static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInTxn) {
-        if (tqDupIntxnReject(pMeta->tqConfigFlag)) {
-          terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN;
-          return -1;
-        }
-        if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-          pMeta->pDeleter(pNode->handle.valueInTxn);
-        }
-      }
-      pNode->handle.valueInTxn = value;
-      tqLinkUnpersist(pMeta, pNode);
-      return 0;
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList));
-  if (pNewNode == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return -1;
-  }
-  pNewNode->handle.key = key;
-  pNewNode->handle.valueInTxn = value;
-  pNewNode->next = pMeta->bucket[bucketKey];
-  pMeta->bucket[bucketKey] = pNewNode;
-  tqLinkUnpersist(pMeta, pNewNode);
-  return 0;
-}
-
-int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); }
-
-int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
-  void* vmem = taosMemoryMalloc(vsize);
-  if (vmem == NULL) {
-    terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
-    return -1;
-  }
-  memcpy(vmem, value, vsize);
-  return tqHandlePutImpl(pMeta, key, vmem);
-}
-
-static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-        return pNode->handle.valueInTxn;
-      } else {
-        return NULL;
-      }
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  return NULL;
-}
-
-int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInTxn == NULL) {
-        terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
-        return -1;
-      }
-      if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
-        pMeta->pDeleter(pNode->handle.valueInUse);
-      }
-      pNode->handle.valueInUse = pNode->handle.valueInTxn;
-      pNode->handle.valueInTxn = NULL;
-      tqLinkUnpersist(pMeta, pNode);
-      return 0;
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
-  return -1;
-}
-
-int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInTxn) {
-        if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-          pMeta->pDeleter(pNode->handle.valueInTxn);
-        }
-        pNode->handle.valueInTxn = NULL;
-        tqLinkUnpersist(pMeta, pNode);
-        return 0;
-      }
-      terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
-      return -1;
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
-  return -1;
-}
-
-int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
-        if (pNode->handle.valueInTxn) {
-          pMeta->pDeleter(pNode->handle.valueInTxn);
-        }
-
-        pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
-        tqLinkUnpersist(pMeta, pNode);
-        return 0;
-      }
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
-  return -1;
-}
-
-int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) {
-  int64_t      bucketKey = key & TQ_BUCKET_MASK;
-  STqMetaList* pNode = pMeta->bucket[bucketKey];
-  while (pNode) {
-    if (pNode->handle.key == key) {
-      pNode->handle.valueInUse = TQ_DELETE_TOKEN;
-      tqLinkUnpersist(pMeta, pNode);
-      return 0;
-    } else {
-      pNode = pNode->next;
-    }
-  }
-  terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
-  return -1;
-}
-
-// TODO: clean deleted idx and data from persistent file
-int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; }

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

@@ -217,7 +217,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p
     if (tDecodeIsEnd(&dc)) break;
 
     // decode row
-    if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) {
+    if (tDecodeBinary(&dc, (uint8_t **)&tRow.pRow, &tRow.szRow) < 0) {
       terrno = TSDB_CODE_INVALID_MSG;
       return -1;
     }
@@ -273,7 +273,7 @@ static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pR
 
 static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) {
   if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1;
-  if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1;
+  if (tDecodeBinary(pDecoder, (uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1;
   return 0;
 }
 

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -85,7 +85,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro
   return 0;
 }
 
-int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) {
+int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
   ASSERT(pMsg != NULL);
   // STsdbMeta *    pMeta = pTsdb->tsdbMeta;
   SSubmitMsgIter msgIter = {0};
@@ -150,7 +150,6 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) {
         return -1;
       }
     }
-
   }
 
   if (terrno != TSDB_CODE_SUCCESS) return -1;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -24,26 +24,66 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in
 static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp);
 
-int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
+int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
-#if 0
+  SDecoder dc = {0};
-  SRpcMsg *pMsg;
-  SRpcMsg  *pRpc;
 
-  *version = pVnode->state.processed;
+  switch (pMsg->msgType) {
-  for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
+    case TDMT_VND_CREATE_TABLE: {
-    pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
+      int64_t ctime = taosGetTimestampMs();
-    pRpc = pMsg;
+      int32_t nReqs;
 
-    // set request version
+      tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
-    if (walWrite(pVnode->pWal, pVnode->state.processed++, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) {
+      tStartDecode(&dc);
-      vError("vnode:%d  write wal error since %s", TD_VID(pVnode), terrstr());
+
-      return -1;
+      tDecodeI32v(&dc, &nReqs);
+      for (int32_t iReq = 0; iReq < nReqs; iReq++) {
+        tb_uid_t uid = tGenIdPI64();
+        tStartDecode(&dc);
+
+        tDecodeI32v(&dc, NULL);
+        *(int64_t *)(dc.data + dc.pos) = uid;
+        *(int64_t *)(dc.data + dc.pos + 8) = ctime;
+
+        tEndDecode(&dc);
+      }
+
+      tEndDecode(&dc);
+      tDecoderClear(&dc);
+    } break;
+    case TDMT_VND_SUBMIT: {
+      SSubmitMsgIter msgIter = {0};
+      SSubmitReq    *pSubmitReq = (SSubmitReq *)pMsg->pCont;
+      SSubmitBlk    *pBlock = NULL;
+      int64_t        ctime = taosGetTimestampMs();
+      tb_uid_t       uid;
+
+      tInitSubmitMsgIter(pSubmitReq, &msgIter);
+
+      for (;;) {
+        tGetSubmitMsgNext(&msgIter, &pBlock);
+        if (pBlock == NULL) break;
+
+        if (msgIter.schemaLen > 0) {
+          uid = tGenIdPI64();
+
+          tDecoderInit(&dc, pBlock->data, msgIter.schemaLen);
+          tStartDecode(&dc);
+
+          tDecodeI32v(&dc, NULL);
+          *(int64_t *)(dc.data + dc.pos) = uid;
+          *(int64_t *)(dc.data + dc.pos + 8) = ctime;
+          pBlock->uid = htobe64(uid);
+
+          tEndDecode(&dc);
+          tDecoderClear(&dc);
         }
       }
 
-  walFsync(pVnode->pWal, false);
+    } break;
+    default:
+      break;
+  }
 
-#endif
   return 0;
 }
 
@@ -106,13 +146,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
                               pMsg->contLen - sizeof(SMsgHead)) < 0) {
       }
     } break;
-#if 0
-    case TDMT_VND_TASK_WRITE_EXEC: {
-      if (tqProcessTaskExec(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead),
-                            0) < 0) {
-      }
-    } break;
-#endif
     case TDMT_VND_ALTER_VNODE:
       break;
     default:
@@ -195,17 +228,6 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
     case TDMT_VND_TASK_RECOVER_RSP:
       return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);
 
-#if 0
-    case TDMT_VND_TASK_PIPE_EXEC:
-    case TDMT_VND_TASK_MERGE_EXEC:
-      return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0);
-    case TDMT_VND_STREAM_TRIGGER:{
-      // refactor, avoid double free
-      int code = tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0);
-      pMsg->pCont = NULL;
-      return code;
-    }
-#endif
     case TDMT_VND_QUERY_HEARTBEAT:
       return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg);
     default:
@@ -675,7 +697,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
     goto _exit;
   }
 
-  for (int i = 0;;) {
+  for (;;) {
     tGetSubmitMsgNext(&msgIter, &pBlock);
     if (pBlock == NULL) break;
 

source/dnode/vnode/test/tqMetaTest.cpp

@@ -1,279 +0,0 @@
-#include <gtest/gtest.h>
-#include <cstring>
-#include <iostream>
-#include <queue>
-
-#include "tqMetaStore.h"
-
-struct Foo {
-  int32_t a;
-};
-
-int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
-  Foo* foo = (Foo*)pObj;
-  if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
-    *ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
-    (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
-  }
-  *(int32_t*)(*ppHead)->content = foo->a;
-  return (*ppHead)->ssize;
-}
-
-const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
-  if (*ppObj == NULL) {
-    *ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t));
-  }
-  Foo* pFoo = *(Foo**)ppObj;
-  pFoo->a = *(int32_t*)pHead->content;
-  return NULL;
-}
-
-void FooDeleter(void* pObj) { taosMemoryFree(pObj); }
-
-class TqMetaUpdateAppendTest : public ::testing::Test {
- protected:
-  void SetUp() override {
-    taosRemoveDir(pathName);
-    pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-    ASSERT(pMeta);
-  }
-
-  void TearDown() override { tqStoreClose(pMeta); }
-
-  STqMetaStore* pMeta;
-  const char*   pathName = TD_TMP_DIR_PATH "tq_test";
-};
-
-TEST_F(TqMetaUpdateAppendTest, copyPutTest) {
-  Foo foo;
-  foo.a = 3;
-  tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo));
-
-  Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-
-  tqHandleCommit(pMeta, 1);
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo->a, 3);
-}
-
-TEST_F(TqMetaUpdateAppendTest, persistTest) {
-  Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
-  pFoo->a = 2;
-  tqHandleMovePut(pMeta, 1, pFoo);
-  Foo* pBar = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pBar == NULL, true);
-  tqHandleCommit(pMeta, 1);
-  pBar = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pBar->a, pFoo->a);
-  pBar = (Foo*)tqHandleGet(pMeta, 2);
-  EXPECT_EQ(pBar == NULL, true);
-
-  tqStoreClose(pMeta);
-  pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-  ASSERT(pMeta);
-
-  pBar = (Foo*)tqHandleGet(pMeta, 1);
-  ASSERT_EQ(pBar != NULL, true);
-  EXPECT_EQ(pBar->a, 2);
-
-  pBar = (Foo*)tqHandleGet(pMeta, 2);
-  EXPECT_EQ(pBar == NULL, true);
-}
-
-TEST_F(TqMetaUpdateAppendTest, uncommittedTest) {
-  Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
-  pFoo->a = 3;
-  tqHandleMovePut(pMeta, 1, pFoo);
-
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-}
-
-TEST_F(TqMetaUpdateAppendTest, abortTest) {
-  Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
-  pFoo->a = 3;
-  tqHandleMovePut(pMeta, 1, pFoo);
-
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-
-  tqHandleAbort(pMeta, 1);
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-}
-
-TEST_F(TqMetaUpdateAppendTest, deleteTest) {
-  Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
-  pFoo->a = 3;
-  tqHandleMovePut(pMeta, 1, pFoo);
-
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-
-  tqHandleCommit(pMeta, 1);
-
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  ASSERT_EQ(pFoo != NULL, true);
-  EXPECT_EQ(pFoo->a, 3);
-
-  tqHandleDel(pMeta, 1);
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  ASSERT_EQ(pFoo != NULL, true);
-  EXPECT_EQ(pFoo->a, 3);
-
-  tqHandleCommit(pMeta, 1);
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-
-  tqStoreClose(pMeta);
-  pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-  ASSERT(pMeta);
-
-  pFoo = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo == NULL, true);
-}
-
-TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
-  Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo));
-  pFoo->a = 3;
-  tqHandleMovePut(pMeta, 1, pFoo);
-  tqHandleCommit(pMeta, 1);
-
-  Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo));
-  pBar->a = 4;
-  tqHandleMovePut(pMeta, 1, pBar);
-
-  Foo* pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo1->a, 3);
-
-  tqStoreClose(pMeta);
-  pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-  ASSERT(pMeta);
-
-  pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo1->a, 3);
-
-  tqHandleCommit(pMeta, 1);
-
-  pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo1->a, 4);
-
-  tqStoreClose(pMeta);
-  pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-  ASSERT(pMeta);
-
-  pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
-  EXPECT_EQ(pFoo1->a, 4);
-}
-
-TEST_F(TqMetaUpdateAppendTest, multiplePage) {
-  taosSeedRand(0);
-  std::vector<int> v;
-  for (int i = 0; i < 1000; i++) {
-    v.push_back(taosRand());
-    Foo foo;
-    foo.a = v[i];
-    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
-  }
-  for (int i = 0; i < 500; i++) {
-    tqHandleCommit(pMeta, i);
-    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
-    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
-    EXPECT_EQ(pFoo->a, v[i]);
-  }
-
-  tqStoreClose(pMeta);
-  pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-  ASSERT(pMeta);
-
-  for (int i = 500; i < 1000; i++) {
-    tqHandleCommit(pMeta, i);
-    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
-    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
-    EXPECT_EQ(pFoo->a, v[i]);
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
-    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
-    EXPECT_EQ(pFoo->a, v[i]);
-  }
-}
-
-TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
-  taosSeedRand(0);
-  std::vector<int> v;
-  for (int i = 0; i < 1000; i++) {
-    v.push_back(taosRand());
-    Foo foo;
-    foo.a = v[i];
-    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
-  }
-
-  for (int i = 0; i < 500; i++) {
-    tqHandleCommit(pMeta, i);
-    v[i] = taosRand();
-    Foo foo;
-    foo.a = v[i];
-    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
-  }
-
-  for (int i = 500; i < 1000; i++) {
-    v[i] = taosRand();
-    Foo foo;
-    foo.a = v[i];
-    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    tqHandleCommit(pMeta, i);
-  }
-
-  tqStoreClose(pMeta);
-  pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
-  ASSERT(pMeta);
-
-  for (int i = 500; i < 1000; i++) {
-    v[i] = taosRand();
-    Foo foo;
-    foo.a = v[i];
-    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
-    tqHandleCommit(pMeta, i);
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
-    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
-    EXPECT_EQ(pFoo->a, v[i]);
-  }
-}
-
-TEST_F(TqMetaUpdateAppendTest, dupCommit) {
-  taosSeedRand(0);
-  std::vector<int> v;
-  for (int i = 0; i < 1000; i++) {
-    v.push_back(taosRand());
-    Foo foo;
-    foo.a = v[i];
-    tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    int ret = tqHandleCommit(pMeta, i);
-    EXPECT_EQ(ret, 0);
-    ret = tqHandleCommit(pMeta, i);
-    EXPECT_EQ(ret, -1);
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    int ret = tqHandleCommit(pMeta, i);
-    EXPECT_EQ(ret, -1);
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
-    ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
-    EXPECT_EQ(pFoo->a, v[i]);
-  }
-}

source/libs/executor/inc/indexoperator.h

@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _INDEX_OPERATOR_H
-#define _INDEX_OPERATOR_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "nodes.h"
-#include "tglobal.h"
-
-typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
-
-SIdxFltStatus idxGetFltStatus(SNode *pFilterNode);
-// construct tag filter operator later
-int32_t doFilterTag(const SNode *pFilterNode, SArray *result);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*INDEX_OPERATOR_*/

source/libs/function/src/builtins.c

@@ -14,8 +14,8 @@
  */
 
 #include "builtins.h"
-#include "querynodes.h"
 #include "builtinsimpl.h"
+#include "querynodes.h"
 #include "scalar.h"
 #include "taoserror.h"
 #include "tdatablock.h"
@@ -185,6 +185,19 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
   if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
+
+  SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, 1);
+  if (nodeType(pParamNode) != QUERY_NODE_VALUE) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+  
+  SValueNode* pValue = (SValueNode*)pParamNode;
+  if (pValue->datum.i < 0 || pValue->datum.i > 100) {
+    return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  pValue->notReserved = true;
+  
   if (3 == paraNum) {
     SNode* pPara3 = nodesListGetNode(pFunc->pParameterList, 2);
     if (QUERY_NODE_VALUE != nodeType(pPara3) || !validAperventileAlgo((SValueNode*)pPara3)) {
@@ -224,6 +237,8 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
+  pValue->notReserved = true;
+
   SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
   pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type};
   return TSDB_CODE_SUCCESS;
@@ -500,8 +515,7 @@ static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
 
   SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
   if (QUERY_NODE_COLUMN != nodeType(pPara)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE can only be columns");
-                           "The parameters of UNIQUE can only be columns");
   }
 
   pFunc->node.resType = ((SExprNode*)pPara)->resType;

source/libs/function/src/tudf.c

@@ -72,12 +72,20 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) {
   char path[PATH_MAX] = {0};
   if (tsProcPath == NULL) {
     path[0] = '.';
+  #ifdef WINDOWS
+    GetModuleFileName(NULL, path, PATH_MAX);
+    taosDirName(path);
+  #endif
   } else {
     strncpy(path, tsProcPath, strlen(tsProcPath));
     taosDirName(path);
   }
 #ifdef WINDOWS
+  if (strlen(path)==0) {
     strcat(path, "udfd.exe");
+  } else {
+    strcat(path, "\\udfd.exe");
+  }
 #else
   strcat(path, "/udfd");
 #endif

source/libs/index/CMakeLists.txt

@@ -12,6 +12,9 @@ target_link_libraries(
     PUBLIC os
     PUBLIC util
     PUBLIC common
+    PUBLIC nodes
+    PUBLIC scalar 
+    PUBLIC function 
 )
 
 if (${BUILD_WITH_LUCENE})

source/libs/index/src/indexFilter.c

@@ -13,10 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "indexoperator.h"
-#include "executorimpl.h"
 #include "index.h"
+#include "indexInt.h"
 #include "nodes.h"
+#include "querynodes.h"
+#include "scalar.h"
 #include "tdatablock.h"
 
 // clang-format off
@@ -69,9 +70,9 @@ typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output)
 static sif_func_t sifNullFunc = NULL;
 // typedef struct SIFWalkParm
 // construct tag filter operator later
-static void destroyTagFilterOperatorInfo(void *param) {
+// static void destroyTagFilterOperatorInfo(void *param) {
-  STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param;
+//  STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param;
-}
+//}
 
 static void sifFreeParam(SIFParam *param) {
   if (param == NULL) return;
@@ -178,13 +179,13 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
     case QUERY_NODE_NODE_LIST: {
       SNodeListNode *nl = (SNodeListNode *)node;
       if (LIST_LENGTH(nl->pNodeList) <= 0) {
-        qError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList));
+        indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList));
         SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
       SIF_ERR_RET(scalarGenerateSetFromList((void **)&param->pFilter, node, nl->dataType.type));
       if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
         taosHashCleanup(param->pFilter);
-        qError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
+        indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
         SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
       break;
@@ -194,7 +195,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
     case QUERY_NODE_LOGIC_CONDITION: {
       SIFParam *res = (SIFParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES);
       if (NULL == res) {
-        qError("no result for node, type:%d, node:%p", nodeType(node), node);
+        indexError("no result for node, type:%d, node:%p", nodeType(node), node);
         SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
       }
       *param = *res;
@@ -210,7 +211,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx
   int32_t code = 0;
   int32_t nParam = sifGetOperParamNum(node->opType);
   if (NULL == node->pLeft || (nParam == 2 && NULL == node->pRight)) {
-    qError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight);
+    indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight);
     SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
   SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam));
@@ -232,7 +233,7 @@ static int32_t sifInitParamList(SIFParam **params, SNodeList *nodeList, SIFCtx *
   int32_t   code = 0;
   SIFParam *tParams = taosMemoryCalloc(nodeList->length, sizeof(SIFParam));
   if (tParams == NULL) {
-    qError("failed to calloc, nodeList: %p", nodeList);
+    indexError("failed to calloc, nodeList: %p", nodeList);
     SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
@@ -252,7 +253,7 @@ _return:
   SIF_RET(code);
 }
 static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *output) {
-  qError("index-filter not support buildin function");
+  indexError("index-filter not support buildin function");
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
@@ -390,7 +391,7 @@ _return:
 
 static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *output) {
   if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
-    qError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList,
+    indexError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList,
                node->pParameterList ? node->pParameterList->length : 0);
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
@@ -485,7 +486,7 @@ EDealRes sifCalcWalker(SNode *node, void *context) {
     return sifWalkOper(node, ctx);
   }
 
-  qError("invalid node type for index filter calculating, type:%d", nodeType(node));
+  indexError("invalid node type for index filter calculating, type:%d", nodeType(node));
   ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
   return DEAL_RES_ERROR;
 }
@@ -509,7 +510,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
   SIFCtx  ctx = {.code = 0, .noExec = false};
   ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   if (NULL == ctx.pRes) {
-    qError("index-filter failed to taosHashInit");
+    indexError("index-filter failed to taosHashInit");
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
@@ -519,7 +520,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
   if (pDst) {
     SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES);
     if (res == NULL) {
-      qError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode));
+      indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode));
       SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
     }
     taosArrayAddAll(pDst->result, res->result);
@@ -539,7 +540,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
   SIFCtx ctx = {.code = 0, .noExec = true};
   ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
   if (NULL == ctx.pRes) {
-    qError("index-filter failed to taosHashInit");
+    indexError("index-filter failed to taosHashInit");
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
@@ -549,7 +550,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) {
 
   SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES);
   if (res == NULL) {
-    qError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode));
+    indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode));
     SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
   *status = res->status;

source/libs/nodes/src/nodesCloneFuncs.c

@@ -19,6 +19,21 @@
 #include "taos.h"
 #include "taoserror.h"
 
+#define COPY_SCALAR_FIELD(fldname)     \
+  do {                                 \
+    (pDst)->fldname = (pSrc)->fldname; \
+  } while (0)
+
+#define COPY_CHAR_ARRAY_FIELD(fldname)        \
+  do {                                        \
+    strcpy((pDst)->fldname, (pSrc)->fldname); \
+  } while (0)
+
+#define COPY_OBJECT_FIELD(fldname, size)                  \
+  do {                                                    \
+    memcpy(&((pDst)->fldname), &((pSrc)->fldname), size); \
+  } while (0)
+
 #define COPY_CHAR_POINT_FIELD(fldname)         \
   do {                                         \
     if (NULL == (pSrc)->fldname) {             \
@@ -70,27 +85,61 @@
     }                                                               \
   } while (0)
 
-static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {}
-
 static SNode* exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
-  dataTypeCopy(&pSrc->resType, &pDst->resType);
+  COPY_OBJECT_FIELD(resType, sizeof(SDataType));
-  pDst->pAssociation = NULL;
+  COPY_CHAR_ARRAY_FIELD(aliasName);
+  COPY_CHAR_ARRAY_FIELD(userAlias);
   return (SNode*)pDst;
 }
 
 static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
-  pDst->pProjectRef = NULL;
+  COPY_SCALAR_FIELD(tableId);
+  COPY_SCALAR_FIELD(tableType);
+  COPY_SCALAR_FIELD(colId);
+  COPY_SCALAR_FIELD(colType);
+  COPY_CHAR_ARRAY_FIELD(dbName);
+  COPY_CHAR_ARRAY_FIELD(tableName);
+  COPY_CHAR_ARRAY_FIELD(tableAlias);
+  COPY_CHAR_ARRAY_FIELD(colName);
+  COPY_SCALAR_FIELD(dataBlockId);
+  COPY_SCALAR_FIELD(slotId);
   return (SNode*)pDst;
 }
 
 static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
   COPY_CHAR_POINT_FIELD(literal);
+  COPY_SCALAR_FIELD(isDuration);
+  COPY_SCALAR_FIELD(translate);
+  COPY_SCALAR_FIELD(notReserved);
+  COPY_SCALAR_FIELD(placeholderNo);
+  COPY_SCALAR_FIELD(typeData);
+  COPY_SCALAR_FIELD(unit);
   if (!pSrc->translate) {
     return (SNode*)pDst;
   }
   switch (pSrc->node.resType.type) {
+    case TSDB_DATA_TYPE_BOOL:
+      COPY_SCALAR_FIELD(datum.b);
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+    case TSDB_DATA_TYPE_SMALLINT:
+    case TSDB_DATA_TYPE_INT:
+    case TSDB_DATA_TYPE_BIGINT:
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      COPY_SCALAR_FIELD(datum.i);
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+    case TSDB_DATA_TYPE_DOUBLE:
+      COPY_SCALAR_FIELD(datum.d);
+      break;
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_UBIGINT:
+      COPY_SCALAR_FIELD(datum.u);
+      break;
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
@@ -104,7 +153,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
     case TSDB_DATA_TYPE_JSON:
     case TSDB_DATA_TYPE_DECIMAL:
     case TSDB_DATA_TYPE_BLOB:
-      // todo
+    case TSDB_DATA_TYPE_MEDIUMBLOB:
     default:
       break;
   }
@@ -113,6 +162,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
 
 static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
+  COPY_SCALAR_FIELD(opType);
   CLONE_NODE_FIELD(pLeft);
   CLONE_NODE_FIELD(pRight);
   return (SNode*)pDst;
@@ -120,18 +170,27 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
 
 static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
+  COPY_SCALAR_FIELD(condType);
   CLONE_NODE_LIST_FIELD(pParameterList);
   return (SNode*)pDst;
 }
 
 static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
+  COPY_CHAR_ARRAY_FIELD(functionName);
+  COPY_SCALAR_FIELD(funcId);
+  COPY_SCALAR_FIELD(funcType);
   CLONE_NODE_LIST_FIELD(pParameterList);
+  COPY_SCALAR_FIELD(udfBufSize);
   return (SNode*)pDst;
 }
 
 static SNode* tableNodeCopy(const STableNode* pSrc, STableNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
+  COPY_CHAR_ARRAY_FIELD(dbName);
+  COPY_CHAR_ARRAY_FIELD(tableName);
+  COPY_CHAR_ARRAY_FIELD(tableAlias);
+  COPY_SCALAR_FIELD(precision);
   return (SNode*)pDst;
 }
 
@@ -159,6 +218,8 @@ static SNode* realTableNodeCopy(const SRealTableNode* pSrc, SRealTableNode* pDst
   COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
   CLONE_OBJECT_FIELD(pMeta, tableMetaClone);
   CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
+  COPY_CHAR_ARRAY_FIELD(qualDbName);
+  COPY_SCALAR_FIELD(ratio);
   return (SNode*)pDst;
 }
 
@@ -170,6 +231,7 @@ static SNode* tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDst
 
 static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) {
   COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
+  COPY_SCALAR_FIELD(joinType);
   CLONE_NODE_FIELD(pLeft);
   CLONE_NODE_FIELD(pRight);
   CLONE_NODE_FIELD(pOnCond);
@@ -177,21 +239,30 @@ static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst
 }
 
 static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
+  COPY_SCALAR_FIELD(dataBlockId);
+  COPY_SCALAR_FIELD(slotId);
   CLONE_NODE_FIELD(pExpr);
   return (SNode*)pDst;
 }
 
 static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) {
+  COPY_SCALAR_FIELD(groupingSetType);
   CLONE_NODE_LIST_FIELD(pParameterList);
   return (SNode*)pDst;
 }
 
 static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode* pDst) {
   CLONE_NODE_FIELD(pExpr);
+  COPY_SCALAR_FIELD(order);
+  COPY_SCALAR_FIELD(nullOrder);
   return (SNode*)pDst;
 }
 
-static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) { return (SNode*)pDst; }
+static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) {
+  COPY_SCALAR_FIELD(limit);
+  COPY_SCALAR_FIELD(offset);
+  return (SNode*)pDst;
+}
 
 static SNode* stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNode* pDst) {
   CLONE_NODE_FIELD(pCol);
@@ -215,13 +286,16 @@ static SNode* intervalWindowNodeCopy(const SIntervalWindowNode* pSrc, SIntervalW
 }
 
 static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) {
+  COPY_OBJECT_FIELD(dataType, sizeof(SDataType));
   CLONE_NODE_LIST_FIELD(pNodeList);
   return (SNode*)pDst;
 }
 
 static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) {
+  COPY_SCALAR_FIELD(mode);
   CLONE_NODE_FIELD(pValues);
   CLONE_NODE_FIELD(pWStartTs);
+  COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
   return (SNode*)pDst;
 }
 
@@ -229,7 +303,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
   CLONE_NODE_LIST_FIELD(pTargets);
   CLONE_NODE_FIELD(pConditions);
   CLONE_NODE_LIST_FIELD(pChildren);
-  pDst->pParent = NULL;
+  COPY_SCALAR_FIELD(optimizedFlag);
   return (SNode*)pDst;
 }
 
@@ -239,12 +313,25 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   CLONE_NODE_LIST_FIELD(pScanPseudoCols);
   CLONE_OBJECT_FIELD(pMeta, tableMetaClone);
   CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
+  COPY_SCALAR_FIELD(scanType);
+  COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2);
+  COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow));
+  COPY_OBJECT_FIELD(tableName, sizeof(SName));
+  COPY_SCALAR_FIELD(showRewrite);
+  COPY_SCALAR_FIELD(ratio);
   CLONE_NODE_LIST_FIELD(pDynamicScanFuncs);
+  COPY_SCALAR_FIELD(dataRequired);
+  COPY_SCALAR_FIELD(interval);
+  COPY_SCALAR_FIELD(offset);
+  COPY_SCALAR_FIELD(sliding);
+  COPY_SCALAR_FIELD(intervalUnit);
+  COPY_SCALAR_FIELD(slidingUnit);
   return (SNode*)pDst;
 }
 
 static SNode* logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  COPY_SCALAR_FIELD(joinType);
   CLONE_NODE_FIELD(pOnConditions);
   return (SNode*)pDst;
 }
@@ -259,32 +346,50 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
 static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pProjections);
+  COPY_CHAR_ARRAY_FIELD(stmtName);
+  COPY_SCALAR_FIELD(limit);
+  COPY_SCALAR_FIELD(offset);
+  COPY_SCALAR_FIELD(slimit);
+  COPY_SCALAR_FIELD(soffset);
   return (SNode*)pDst;
 }
 
 static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
-  pDst->pDataBlocks = NULL;
+  COPY_SCALAR_FIELD(msgType);
-  pDst->pVgDataBlocks = NULL;
   return (SNode*)pDst;
 }
 
 static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  COPY_SCALAR_FIELD(srcGroupId);
+  COPY_SCALAR_FIELD(precision);
   return (SNode*)pDst;
 }
 
 static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  COPY_SCALAR_FIELD(winType);
   CLONE_NODE_LIST_FIELD(pFuncs);
+  COPY_SCALAR_FIELD(interval);
+  COPY_SCALAR_FIELD(offset);
+  COPY_SCALAR_FIELD(sliding);
+  COPY_SCALAR_FIELD(intervalUnit);
+  COPY_SCALAR_FIELD(slidingUnit);
+  COPY_SCALAR_FIELD(sessionGap);
   CLONE_NODE_FIELD(pTspk);
+  CLONE_NODE_FIELD(pStateExpr);
+  COPY_SCALAR_FIELD(triggerType);
+  COPY_SCALAR_FIELD(watermark);
   return (SNode*)pDst;
 }
 
 static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  COPY_SCALAR_FIELD(mode);
   CLONE_NODE_FIELD(pWStartTs);
   CLONE_NODE_FIELD(pValues);
+  COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
   return (SNode*)pDst;
 }
 
@@ -301,28 +406,41 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi
 }
 
 static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
+  COPY_OBJECT_FIELD(id, sizeof(SSubplanId));
   CLONE_NODE_FIELD(pNode);
-  pDst->pChildren = NULL;
+  COPY_SCALAR_FIELD(subplanType);
-  pDst->pParents = NULL;
+  COPY_SCALAR_FIELD(level);
-  pDst->pVgroupList = NULL;
+  COPY_SCALAR_FIELD(splitFlag);
   return (SNode*)pDst;
 }
 
 static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
+  COPY_SCALAR_FIELD(dataBlockId);
   CLONE_NODE_LIST_FIELD(pSlots);
+  COPY_SCALAR_FIELD(totalRowSize);
+  COPY_SCALAR_FIELD(outputRowSize);
+  COPY_SCALAR_FIELD(precision);
   return (SNode*)pDst;
 }
 
 static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) {
-  dataTypeCopy(&pSrc->dataType, &pDst->dataType);
+  COPY_SCALAR_FIELD(slotId);
+  COPY_OBJECT_FIELD(dataType, sizeof(SDataType));
+  COPY_SCALAR_FIELD(reserve);
+  COPY_SCALAR_FIELD(output);
+  COPY_SCALAR_FIELD(tag);
   return (SNode*)pDst;
 }
 
 static SNode* downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) {
+  COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr));
+  COPY_SCALAR_FIELD(taskId);
+  COPY_SCALAR_FIELD(schedId);
   return (SNode*)pDst;
 }
 
 static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
+  COPY_SCALAR_FIELD(isDistinct);
   CLONE_NODE_LIST_FIELD(pProjectionList);
   CLONE_NODE_FIELD(pFromTable);
   CLONE_NODE_FIELD(pWhere);
@@ -333,6 +451,12 @@ static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
   CLONE_NODE_LIST_FIELD(pOrderByList);
   CLONE_NODE_FIELD(pLimit);
   CLONE_NODE_FIELD(pLimit);
+  COPY_CHAR_ARRAY_FIELD(stmtName);
+  COPY_SCALAR_FIELD(precision);
+  COPY_SCALAR_FIELD(isEmptyResult);
+  COPY_SCALAR_FIELD(isTimeOrderQuery);
+  COPY_SCALAR_FIELD(hasAggFuncs);
+  COPY_SCALAR_FIELD(hasRepeatScanFuncs);
   return (SNode*)pDst;
 }
 
@@ -345,7 +469,6 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  memcpy(pDst, pNode, nodesNodeSize(nodeType(pNode)));
   switch (nodeType(pNode)) {
     case QUERY_NODE_COLUMN:
       return columnNodeCopy((const SColumnNode*)pNode, (SColumnNode*)pDst);
@@ -387,6 +510,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
       return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst);
     case QUERY_NODE_DOWNSTREAM_SOURCE:
       return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst);
+    case QUERY_NODE_LEFT_VALUE:
+      return pDst;
     case QUERY_NODE_SELECT_STMT:
       return selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst);
     case QUERY_NODE_LOGIC_PLAN_SCAN:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -78,6 +78,8 @@ const char* nodesNodeName(ENodeType type) {
       return "TableOptions";
     case QUERY_NODE_INDEX_OPTIONS:
       return "IndexOptions";
+    case QUERY_NODE_LEFT_VALUE:
+      return "LeftValue";
     case QUERY_NODE_SET_OPERATOR:
       return "SetOperator";
     case QUERY_NODE_SELECT_STMT:
@@ -2175,7 +2177,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) {
         code = TSDB_CODE_OUT_OF_MEMORY;
         break;
       }
-      varDataSetLen(pNode->datum.p, pNode->node.resType.bytes);
+      varDataSetLen(pNode->datum.p, pNode->node.resType.bytes - VARSTR_HEADER_SIZE);
       if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) {
         char* buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1);
         if (NULL == buf) {
@@ -3019,6 +3021,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       break;
     case QUERY_NODE_DOWNSTREAM_SOURCE:
       return downstreamSourceNodeToJson(pObj, pJson);
+    case QUERY_NODE_LEFT_VALUE:
+      return TSDB_CODE_SUCCESS;  // SLeftValueNode has no fields to serialize.
     case QUERY_NODE_SET_OPERATOR:
       return setOperatorToJson(pObj, pJson);
     case QUERY_NODE_SELECT_STMT:
@@ -3130,6 +3134,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToSlotDescNode(pJson, pObj);
     case QUERY_NODE_DOWNSTREAM_SOURCE:
       return jsonToDownstreamSourceNode(pJson, pObj);
+    case QUERY_NODE_LEFT_VALUE:
+      return TSDB_CODE_SUCCESS;  // SLeftValueNode has no fields to deserialize.
     case QUERY_NODE_SET_OPERATOR:
       return jsonToSetOperator(pJson, pObj);
     case QUERY_NODE_SELECT_STMT:

source/libs/nodes/src/nodesUtilFuncs.c

@@ -79,6 +79,8 @@ int32_t nodesNodeSize(ENodeType type) {
       return sizeof(SStreamOptions);
     case QUERY_NODE_TOPIC_OPTIONS:
       return sizeof(STopicOptions);
+    case QUERY_NODE_LEFT_VALUE:
+      return sizeof(SLeftValueNode);
     case QUERY_NODE_SET_OPERATOR:
       return sizeof(SSetOperator);
     case QUERY_NODE_SELECT_STMT:

source/libs/parser/src/parTranslater.c

@@ -646,12 +646,13 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
       }
       case TSDB_DATA_TYPE_VARCHAR:
       case TSDB_DATA_TYPE_VARBINARY: {
-       pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1);
+        pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1);
         if (NULL == pVal->datum.p) {
           return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
         }
-       varDataSetLen(pVal->datum.p, targetDt.bytes);
+        int32_t len = TMIN(targetDt.bytes - VARSTR_HEADER_SIZE, pVal->node.resType.bytes);
-       strncpy(varDataVal(pVal->datum.p), pVal->literal, targetDt.bytes);
+        varDataSetLen(pVal->datum.p, len);
+        strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
         break;
       }
       case TSDB_DATA_TYPE_TIMESTAMP: {
@@ -662,19 +663,18 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
         break;
       }
       case TSDB_DATA_TYPE_NCHAR: {
-       int32_t bytes = targetDt.bytes * TSDB_NCHAR_SIZE;
+        pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1);
-       pVal->datum.p = taosMemoryCalloc(1, bytes + VARSTR_HEADER_SIZE + 1);
         if (NULL == pVal->datum.p) {
           return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
           ;
         }
 
-       int32_t output = 0;
+        int32_t len = 0;
-       if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), bytes,
+        if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p),
-                          &output)) {
+                           targetDt.bytes - VARSTR_HEADER_SIZE, &len)) {
           return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
         }
-       varDataSetLen(pVal->datum.p, output);
+        varDataSetLen(pVal->datum.p, len);
         break;
       }
       case TSDB_DATA_TYPE_DECIMAL:
@@ -689,8 +689,20 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD
   return DEAL_RES_CONTINUE;
 }
 
+static int32_t calcTypeBytes(SDataType dt) {
+  if (TSDB_DATA_TYPE_BINARY == dt.type) {
+    return dt.bytes + VARSTR_HEADER_SIZE;
+  } else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
+    return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
+  } else {
+    return dt.bytes;
+  }
+}
+
 static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
- return translateValueImpl(pCxt, pVal, pVal->node.resType);
+  SDataType dt = pVal->node.resType;
+  dt.bytes = calcTypeBytes(dt);
+  return translateValueImpl(pCxt, pVal, dt);
 }
 
 static bool isMultiResFunc(SNode* pNode) {
@@ -2341,16 +2353,6 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm
  return buildCmdMsg(pCxt, TDMT_MND_ALTER_DB, (FSerializeFunc)tSerializeSAlterDbReq, &alterReq);
 }
 
-static int32_t calcTypeBytes(SDataType dt) {
- if (TSDB_DATA_TYPE_BINARY == dt.type) {
-   return dt.bytes + VARSTR_HEADER_SIZE;
- } else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
-   return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
- } else {
-   return dt.bytes;
- }
-}
-
 static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) {
  *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField));
  SNode* pNode;
@@ -4089,18 +4091,8 @@ static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* p
  return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal);
 }
 
-static int32_t colDataBytesToValueDataBytes(uint8_t type, int32_t bytes) {
- if (TSDB_DATA_TYPE_VARCHAR == type || TSDB_DATA_TYPE_BINARY == type || TSDB_DATA_TYPE_VARBINARY == type) {
-   return bytes - VARSTR_HEADER_SIZE;
- } else if (TSDB_DATA_TYPE_NCHAR == type) {
-   return (bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
- }
- return bytes;
-}
-
 static SDataType schemaToDataType(SSchema* pSchema) {
   SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0};
- dt.bytes = colDataBytesToValueDataBytes(pSchema->type, pSchema->bytes);
   return dt;
 }
 

source/libs/parser/test/parInitialATest.cpp

@@ -204,7 +204,7 @@ TEST_F(ParserInitialATest, alterTable) {
     }
   };
 
-  auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, const uint8_t* pNewVal, uint32_t bytes) {
+  auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) {
     memset(&expect, 0, sizeof(SVAlterTbReq));
     expect.tbName = strdup(pTbname);
     expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL;
@@ -215,7 +215,7 @@ TEST_F(ParserInitialATest, alterTable) {
     expect.pTagVal = pNewVal;
   };
 
-  auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, const char* pComment = nullptr) {
+  auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) {
     memset(&expect, 0, sizeof(SVAlterTbReq));
     expect.tbName = strdup(pTbname);
     expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS;
@@ -240,7 +240,7 @@ TEST_F(ParserInitialATest, alterTable) {
     void*          pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead));
     SVAlterTbReq   req = {0};
     SDecoder       coder = {0};
-    tDecoderInit(&coder, (const uint8_t*)pBuf, pVgData->size);
+    tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size);
     ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS);
 
     ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName));
@@ -274,7 +274,7 @@ TEST_F(ParserInitialATest, alterTable) {
   setAlterOptionsFunc("t1", 10, nullptr);
   run("ALTER TABLE t1 TTL 10");
 
-  setAlterOptionsFunc("t1", -1, "test");
+  setAlterOptionsFunc("t1", -1, (char*)"test");
   run("ALTER TABLE t1 COMMENT 'test'");
 
   setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT);
@@ -290,7 +290,7 @@ TEST_F(ParserInitialATest, alterTable) {
   run("ALTER TABLE t1 RENAME COLUMN c1 cc1");
 
   int32_t val = 10;
-  setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val));
+  setAlterTagFunc("st1s1", "tag1", (uint8_t*)&val, sizeof(val));
   run("ALTER TABLE st1s1 SET TAG tag1=10");
 
   // todo

source/libs/planner/src/planPhysiCreater.c

@@ -599,14 +599,17 @@ typedef struct SRewritePrecalcExprsCxt {
 static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
   SNode* pExpr = nodesCloneNode(*pNode);
   if (NULL == pExpr) {
+    pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
     return DEAL_RES_ERROR;
   }
   if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) {
+    pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
     nodesDestroyNode(pExpr);
     return DEAL_RES_ERROR;
   }
   SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
   if (NULL == pCol) {
+    pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
     nodesDestroyNode(pExpr);
     return DEAL_RES_ERROR;
   }
@@ -624,16 +627,45 @@ static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode)
   return DEAL_RES_IGNORE_CHILD;
 }
 
+static int32_t rewriteValueToOperator(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) {
+  SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR);
+  if (NULL == pOper) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  pOper->pLeft = nodesMakeNode(QUERY_NODE_LEFT_VALUE);
+  if (NULL == pOper->pLeft) {
+    nodesDestroyNode(pOper);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  SValueNode* pVal = (SValueNode*)*pNode;
+  pOper->node.resType = pVal->node.resType;
+  strcpy(pOper->node.aliasName, pVal->node.aliasName);
+  pOper->opType = OP_TYPE_ASSIGN;
+  pOper->pRight = *pNode;
+  *pNode = (SNode*)pOper;
+  return TSDB_CODE_SUCCESS;
+}
+
 static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
   SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext;
   switch (nodeType(*pNode)) {
+    case QUERY_NODE_VALUE: {
+      if (((SValueNode*)*pNode)->notReserved) {
+        break;
+      }
+      pCxt->errCode = rewriteValueToOperator(pCxt, pNode);
+      if (TSDB_CODE_SUCCESS != pCxt->errCode) {
+        return DEAL_RES_ERROR;
+      }
+      return collectAndRewrite(pCxt, pNode);
+    }
     case QUERY_NODE_OPERATOR:
     case QUERY_NODE_LOGIC_CONDITION: {
-      return collectAndRewrite(pContext, pNode);
+      return collectAndRewrite(pCxt, pNode);
     }
     case QUERY_NODE_FUNCTION: {
       if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) {
-        return collectAndRewrite(pContext, pNode);
+        return collectAndRewrite(pCxt, pNode);
       }
     }
     default:
@@ -677,9 +709,8 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN
   }
   SRewritePrecalcExprsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs};
   nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt);
-  if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
+  if (0 == LIST_LENGTH(cxt.pPrecalcExprs) || TSDB_CODE_SUCCESS != cxt.errCode) {
-    nodesDestroyList(cxt.pPrecalcExprs);
+    DESTORY_LIST(*pPrecalcExprs);
-    *pPrecalcExprs = NULL;
   }
   return cxt.errCode;
 }

source/libs/planner/test/planBasicTest.cpp

@@ -50,4 +50,6 @@ TEST_F(PlanBasicTest, func) {
   run("SELECT DIFF(c1) FROM t1");
 
   run("SELECT PERCENTILE(c1, 60) FROM t1");
+
+  run("SELECT TOP(c1, 60) FROM t1");
 }

source/libs/planner/test/planGroupByTest.cpp

@@ -49,6 +49,8 @@ TEST_F(PlanGroupByTest, aggFunc) {
   run("SELECT LAST(*), FIRST(*) FROM t1");
 
   run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1");
+
+  run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2");
 }
 
 TEST_F(PlanGroupByTest, selectFunc) {

source/libs/planner/test/planTestUtil.cpp

@@ -14,6 +14,7 @@
  */
 
 #include "planTestUtil.h"
+#include <getopt.h>
 
 #include <algorithm>
 #include <array>

source/libs/scalar/src/scalar.c

@@ -182,6 +182,11 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) {
 
 int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) {
   switch (nodeType(node)) {
+    case QUERY_NODE_LEFT_VALUE: {
+      SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0);
+      param->numOfRows = pb->info.rows;
+      break;
+    }
     case QUERY_NODE_VALUE: {
       SValueNode *valueNode = (SValueNode *)node;
 
@@ -845,7 +850,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
 }
 
 EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
-  if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
+  if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)|| QUERY_NODE_LEFT_VALUE == nodeType(pNode)) {
     return DEAL_RES_CONTINUE;
   }
 

source/libs/scalar/src/sclfunc.c

@@ -824,7 +824,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
         }
         //for constant conversion, need to set proper length of pOutput description
         if (len < outputLen) {
-          pOutput->columnData->info.bytes = len;
+          pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE;
         }
         break;
       }

source/libs/scalar/src/sclvector.c

@@ -1333,6 +1333,22 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO
   doReleaseVec(pLeftCol,  leftConvert);
 }
 
+void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  pOut->numOfRows = pLeft->numOfRows;
+
+  if (colDataIsNull_s(pRight->columnData, 0)) {
+    for (int32_t i = 0; i < pOut->numOfRows; ++i) {
+      colDataAppend(pOutputCol, i, NULL, true);
+    }
+  } else {
+    for (int32_t i = 0; i < pOut->numOfRows; ++i) {
+      colDataAppend(pOutputCol, i, colDataGetData(pRight->columnData, 0), false);
+    }
+  }
+}
+
 void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) {
 #if 0
   int32_t len = pLeft->bytes + pRight->bytes;
@@ -1691,6 +1707,8 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
       return vectorMathRemainder;
     case OP_TYPE_MINUS:
       return vectorMathMinus;
+    case OP_TYPE_ASSIGN:
+      return vectorAssign;
     case OP_TYPE_GREATER_THAN:
       return vectorGreater;
     case OP_TYPE_GREATER_EQUAL:

source/libs/scheduler/inc/schedulerInt.h

@@ -132,7 +132,7 @@ typedef struct SSchLevel {
   int32_t         taskSucceed;
   int32_t         taskNum;
   int32_t         taskLaunchedNum;
-  SHashObj       *flowCtrl;      // key is ep, element is SSchFlowControl
+  int32_t         taskDoneNum;
   SArray         *subTasks;      // Element is SQueryTask
 } SSchLevel;
 
@@ -175,11 +175,13 @@ typedef struct SSchJob {
   SArray          *levels;    // starting from 0. SArray<SSchLevel>
   SNodeList       *subPlans;  // subplan pointer copied from DAG, no need to free it in scheduler
 
+  SArray          *dataSrcTasks; // SArray<SQueryTask*>
   int32_t          levelIdx;
   SEpSet           dataSrcEps;
   SHashObj        *execTasks; // executing tasks, key:taskid, value:SQueryTask*
   SHashObj        *succTasks; // succeed tasks, key:taskid, value:SQueryTask*
   SHashObj        *failTasks; // failed tasks, key:taskid, value:SQueryTask*
+  SHashObj        *flowCtrl;  // key is ep, element is SSchFlowControl
 
   SExplainCtx     *explainCtx;
   int8_t           status;  
@@ -200,7 +202,7 @@ typedef struct SSchJob {
 
 extern SSchedulerMgmt schMgmt;
 
-#define SCH_TASK_READY_TO_LUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children))
+#define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children))
 
 #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1)
 #define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0)
@@ -223,7 +225,7 @@ extern SSchedulerMgmt schMgmt;
 
 #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true
 #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl)
-#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEAF_TASK(_job, _task) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
+#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
 
 #define SCH_SET_JOB_TYPE(_job, type) (_job)->attr.queryJob = ((type) != SUBPLAN_TYPE_MODIFY)
 #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) 
@@ -261,7 +263,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task);
 int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType);
 SSchJob *schAcquireJob(int64_t refId);
 int32_t schReleaseJob(int64_t refId);
-void schFreeFlowCtrl(SSchLevel *pLevel);
+void schFreeFlowCtrl(SSchJob *pJob);
 int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel);
 int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask);
 int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough);

source/libs/scheduler/src/schFlowCtrl.c

@@ -19,13 +19,13 @@
 #include "catalog.h"
 #include "tref.h"
 
-void schFreeFlowCtrl(SSchLevel *pLevel) {
+void schFreeFlowCtrl(SSchJob *pJob) {
-  if (NULL == pLevel->flowCtrl) {
+  if (NULL == pJob->flowCtrl) {
     return;
   }
 
   SSchFlowControl *ctrl = NULL;
-  void *pIter = taosHashIterate(pLevel->flowCtrl, NULL);
+  void *pIter = taosHashIterate(pJob->flowCtrl, NULL);
   while (pIter) {
     ctrl = (SSchFlowControl *)pIter;
 
@@ -33,11 +33,11 @@ void schFreeFlowCtrl(SSchLevel *pLevel) {
       taosArrayDestroy(ctrl->taskList);
     }
     
-    pIter = taosHashIterate(pLevel->flowCtrl, pIter);
+    pIter = taosHashIterate(pJob->flowCtrl, pIter);
   }
 
-  taosHashCleanup(pLevel->flowCtrl);
+  taosHashCleanup(pJob->flowCtrl);
-  pLevel->flowCtrl = NULL;
+  pJob->flowCtrl = NULL;
 }
 
 int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
@@ -47,9 +47,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
   }
 
   int32_t sum = 0;
-  
+  int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks);
-  for (int32_t i = 0; i < pLevel->taskNum; ++i) {
+  for (int32_t i = 0; i < taskNum; ++i) {
-    SSchTask *pTask = taosArrayGet(pLevel->subTasks, i);
+    SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i);
 
     sum += pTask->plan->execNodeStat.tableNum;
   }
@@ -59,9 +59,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) {
     return TSDB_CODE_SUCCESS;
   }
 
-  pLevel->flowCtrl = taosHashInit(pLevel->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+  pJob->flowCtrl = taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
-  if (NULL == pLevel->flowCtrl) {
+  if (NULL == pJob->flowCtrl) {
-    SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pLevel->taskNum);
+    SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum);
     SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
@@ -78,7 +78,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) {
   int32_t code = 0;
   SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
   
-  ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp));
+  ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp));
   if (NULL == ctrl) {
     SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port);
     SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
@@ -110,11 +110,11 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) {
   SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
   
   do {
-    ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp));
+    ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp));
     if (NULL == ctrl) {
       SSchFlowControl nctrl = {.tableNumSum = pTask->plan->execNodeStat.tableNum, .execTaskNum = 1};
 
-      code = taosHashPut(pLevel->flowCtrl, ep, sizeof(SEp), &nctrl, sizeof(nctrl));
+      code = taosHashPut(pJob->flowCtrl, ep, sizeof(SEp), &nctrl, sizeof(nctrl));
       if (code) {
         if (HASH_NODE_EXIST(code)) {
           continue;
@@ -273,10 +273,9 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) {
 
   SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask));
 
-  SSchLevel *pLevel = pTask->level;
   SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode);
   
-  SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp));
+  SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp));
   if (NULL == ctrl) {
     SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port);
     SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);

source/libs/scheduler/src/scheduler.c

@@ -391,6 +391,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
           SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
           SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
         }
+
+        SCH_TASK_DLOG("children info, the %d child TID %" PRIx64, n, (*childTask)->taskId);
       }
 
       if (parentNum > 0) {
@@ -423,6 +425,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
           SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n);
           SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
         }
+
+        SCH_TASK_DLOG("parents info, the %d parent TID %" PRIx64, n, (*parentTask)->taskId);        
       }
 
       SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum);
@@ -464,6 +468,17 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t schRecordQueryDataSrc(SSchJob *pJob, SSchTask *pTask) {
+  if (!SCH_IS_DATA_SRC_QRY_TASK(pTask)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  taosArrayPush(pJob->dataSrcTasks, &pTask);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
   int32_t code = 0;
   pJob->queryId = pDag->queryId;
@@ -473,6 +488,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
     SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
 
+  pJob->dataSrcTasks = taosArrayInit(pDag->numOfSubplans, POINTER_BYTES);
+  if (NULL == pJob->dataSrcTasks) {
+    SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+  }
+
   int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans);
   if (levelNum <= 0) {
     SCH_JOB_ELOG("invalid level num:%d", levelNum);
@@ -551,6 +571,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
         SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
 
+      SCH_ERR_JRET(schRecordQueryDataSrc(pJob, p));
+
       if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) {
         SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n);
         SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
@@ -629,6 +651,17 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) {
+  int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId));
+  if (code) {
+    SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code);
+    SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) {
   int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES);
   if (0 != code) {
@@ -774,6 +807,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
 int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
   atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
 
+  SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask));
+  SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START);
+  
   if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
     SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask));
     SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask));
@@ -947,6 +983,32 @@ _return:
   SCH_RET(schProcessOnJobFailure(pJob, errCode));
 }
 
+int32_t schLaunchNextLevelTasks(SSchJob *pJob, SSchTask *pTask) {
+  if (!SCH_IS_QUERY_JOB(pJob)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SSchLevel *pLevel = pTask->level;
+  int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1);
+  if (doneNum == pLevel->taskNum) {
+    pJob->levelIdx--;
+
+    pLevel = taosArrayGet(pJob->levels, pJob->levelIdx);
+    for (int32_t i = 0; i < pLevel->taskNum; ++i) {
+      SSchTask *pTask = taosArrayGet(pLevel->subTasks, i);
+
+      if (pTask->children && taosArrayGetSize(pTask->children) > 0) {
+        continue;
+      }
+      
+      SCH_ERR_RET(schLaunchTask(pJob, pTask));
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 // Note: no more task error processing, handled in function internal
 int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
   bool    moved = false;
@@ -1015,11 +1077,13 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
     qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source);
     SCH_UNLOCK(SCH_WRITE, &par->lock);
 
-    if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) {
+    if (SCH_TASK_READY_FOR_LAUNCH(readyNum, par)) {
-      SCH_ERR_RET(schLaunchTaskImpl(pJob, par));
+      SCH_ERR_RET(schLaunchTask(pJob, par));
     }
   }
 
+  SCH_ERR_RET(schLaunchNextLevelTasks(pJob, pTask));
+
   return TSDB_CODE_SUCCESS;
 
 _return:
@@ -2400,8 +2464,6 @@ void schFreeJobImpl(void *job) {
   for (int32_t i = 0; i < numOfLevels; ++i) {
     SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
 
-    schFreeFlowCtrl(pLevel);
-
     int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
     for (int32_t j = 0; j < numOfTasks; ++j) {
       SSchTask *pTask = taosArrayGet(pLevel->subTasks, j);
@@ -2411,12 +2473,15 @@ void schFreeJobImpl(void *job) {
     taosArrayDestroy(pLevel->subTasks);
   }
 
+  schFreeFlowCtrl(pJob);
+
   taosHashCleanup(pJob->execTasks);
   taosHashCleanup(pJob->failTasks);
   taosHashCleanup(pJob->succTasks);
 
   taosArrayDestroy(pJob->levels);
   taosArrayDestroy(pJob->nodeList);
+  taosArrayDestroy(pJob->dataSrcTasks);
 
   qExplainFreeCtx(pJob->explainCtx);
 

source/libs/stream/src/tstream.c

@@ -134,7 +134,7 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
 }
 
 static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
-  void* exec = pTask->exec.runners[0].executor;
+  void* exec = pTask->exec.executor;
 
   // set input
   if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
@@ -171,12 +171,12 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
 }
 
 // TODO: handle version
-int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) {
+int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
   SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
   if (pRes == NULL) return -1;
   while (1) {
     int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
-    void*  exec = pTask->exec.runners[0].executor;
+    void*  exec = pTask->exec.executor;
     if (execStatus == TASK_STATUS__IDLE) {
       // first run, from qall, handle failure from last exec
       while (1) {
@@ -278,7 +278,7 @@ FAIL:
   return -1;
 }
 
-int32_t streamTaskSink(SStreamTask* pTask, SMsgCb* pMsgCb) {
+int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) {
   bool firstRun = 1;
   while (1) {
     SStreamDataBlock* pBlock = NULL;
@@ -407,7 +407,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg*
   return 0;
 }
 
-int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
+int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
   // 1. handle input
   streamTaskEnqueue(pTask, pReq, pRsp);
 
@@ -415,172 +415,42 @@ int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStream
   // 2.1. idle: exec
   // 2.2. executing: return
   // 2.3. closing: keep trying
-  streamTaskExec2(pTask, pMsgCb);
+  streamExec(pTask, pMsgCb);
 
   // 3. handle output
   // 3.1 check and set status
   // 3.2 dispatch / sink
-  streamTaskSink(pTask, pMsgCb);
+  streamSink(pTask, pMsgCb);
 
   return 0;
 }
 
-int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) {
+int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) {
   atomic_store_8(&pTask->inputStatus, pRsp->inputStatus);
   if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) {
     // TODO: init recover timer
   }
   // continue dispatch
-  streamTaskSink(pTask, pMsgCb);
+  streamSink(pTask, pMsgCb);
   return 0;
 }
 
 int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) {
-  streamTaskExec2(pTask, pMsgCb);
+  streamExec(pTask, pMsgCb);
-  streamTaskSink(pTask, pMsgCb);
+  streamSink(pTask, pMsgCb);
   return 0;
 }
 
-int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) {
+int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) {
   //
   return 0;
 }
 
-int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) {
+int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) {
   //
   return 0;
 }
 
-int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) {
-  SArray* pRes = NULL;
-  // source
-  if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK && pTask->sourceType != TASK_SOURCE__SCAN) return 0;
-
-  // exec
-  if (pTask->execType != TASK_EXEC__NONE) {
-    ASSERT(workId < pTask->exec.numOfRunners);
-    void* exec = pTask->exec.runners[workId].executor;
-    pRes = taosArrayInit(0, sizeof(SSDataBlock));
-    if (pRes == NULL) {
-      return -1;
-    }
-    if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
-      qSetStreamInput(exec, input, inputType);
-      while (1) {
-        SSDataBlock* output;
-        uint64_t     ts;
-        if (qExecTask(exec, &output, &ts) < 0) {
-          ASSERT(false);
-        }
-        if (output == NULL) {
-          break;
-        }
-        taosArrayPush(pRes, output);
-      }
-    } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
-      const SArray* blocks = (const SArray*)input;
-      /*int32_t       sz = taosArrayGetSize(blocks);*/
-      /*for (int32_t i = 0; i < sz; i++) {*/
-      /*SSDataBlock* pBlock = taosArrayGet(blocks, i);*/
-      /*qSetStreamInput(exec, pBlock, inputType);*/
-      qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
-      while (1) {
-        SSDataBlock* output;
-        uint64_t     ts;
-        if (qExecTask(exec, &output, &ts) < 0) {
-          ASSERT(false);
-        }
-        if (output == NULL) {
-          break;
-        }
-        taosArrayPush(pRes, output);
-      }
-      /*}*/
-    } else {
-      ASSERT(0);
-    }
-  } else {
-    ASSERT(inputType == STREAM_DATA_TYPE_SSDATA_BLOCK);
-    pRes = (SArray*)input;
-  }
-
-  if (pRes == NULL || taosArrayGetSize(pRes) == 0) return 0;
-
-  // sink
-  if (pTask->sinkType == TASK_SINK__TABLE) {
-    // blockDebugShowData(pRes);
-    pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes);
-  } else if (pTask->sinkType == TASK_SINK__SMA) {
-    pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
-    //
-  } else if (pTask->sinkType == TASK_SINK__FETCH) {
-    //
-  } else {
-    ASSERT(pTask->sinkType == TASK_SINK__NONE);
-  }
-
-  // dispatch
-
-  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
-    SRpcMsg dispatchMsg = {0};
-    if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
-      ASSERT(0);
-      return -1;
-    }
-
-    int32_t qType;
-    if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) {
-      qType = FETCH_QUEUE;
-    } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||
-               pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {
-      qType = MERGE_QUEUE;
-    } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {
-      qType = WRITE_QUEUE;
-    } else {
-      ASSERT(0);
-    }
-    tmsgPutToQueue(pMsgCb, qType, &dispatchMsg);
-
-  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-    SRpcMsg dispatchMsg = {0};
-    SEpSet* pEpSet = NULL;
-    if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
-      ASSERT(0);
-      return -1;
-    }
-
-    tmsgSendReq(pEpSet, &dispatchMsg);
-
-  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
-    SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-    if (pShuffleRes == NULL) {
-      return -1;
-    }
-
-    int32_t sz = taosArrayGetSize(pRes);
-    for (int32_t i = 0; i < sz; i++) {
-      SSDataBlock* pDataBlock = taosArrayGet(pRes, i);
-      SArray*      pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t));
-      if (pArray == NULL) {
-        pArray = taosArrayInit(0, sizeof(SSDataBlock));
-        if (pArray == NULL) {
-          return -1;
-        }
-        taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*));
-      }
-      taosArrayPush(pArray, pDataBlock);
-    }
-
-    if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) {
-      return -1;
-    }
-
-  } else {
-    ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
-  }
-  return 0;
-}
-
 int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI64(buf, pReq->streamId);
@@ -607,20 +477,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
   pTask->streamId = streamId;
   pTask->status = TASK_STATUS__IDLE;
 
-  pTask->inputQ = taosOpenQueue();
-  pTask->outputQ = taosOpenQueue();
-  pTask->inputQAll = taosAllocateQall();
-  pTask->outputQAll = taosAllocateQall();
-  if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
-    goto FAIL;
   return pTask;
-FAIL:
-  if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
-  if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
-  if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
-  if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
-  if (pTask) taosMemoryFree(pTask);
-  return NULL;
 }
 
 int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
@@ -722,11 +579,7 @@ void tFreeSStreamTask(SStreamTask* pTask) {
   taosCloseQueue(pTask->outputQ);
   // TODO
   if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
-  for (int32_t i = 0; i < pTask->exec.numOfRunners; i++) {
+  qDestroyTask(pTask->exec.executor);
-    qDestroyTask(pTask->exec.runners[i].executor);
-  }
-  taosMemoryFree(pTask->exec.runners);
-  /*taosMemoryFree(pTask->executor);*/
   taosMemoryFree(pTask);
 }
 

source/libs/sync/src/syncMessage.c

@@ -411,7 +411,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) {
   }
   uint32_t len;
   char*    data = NULL;
-  if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) {
+  if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) {
     return NULL;
   }
   assert(len = pMsg->dataLen);
@@ -670,7 +670,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) {
   }
   uint32_t len;
   char*    data = NULL;
-  if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) {
+  if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) {
     return NULL;
   }
   assert(len = pMsg->dataLen);

source/os/CMakeLists.txt

@@ -18,15 +18,17 @@ if(USE_TD_MEMORY)
     add_definitions(-DUSE_TD_MEMORY) 
 endif ()
 if(BUILD_ADDR2LINE)
+    if(NOT TD_WINDOWS)
         target_include_directories(
             os
             PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf"
         )
-    add_definitions(-DUSE_ADDR2LINE)
         target_link_libraries(
             os PUBLIC addr2line dl z
         )
     endif()
+    add_definitions(-DUSE_ADDR2LINE)
+endif ()
 if(CHECK_STR2INT_ERROR)
     add_definitions(-DTD_CHECK_STR_TO_INT_ERROR)
 endif()

source/os/src/osDir.c

@@ -91,7 +91,12 @@ void taosRemoveDir(const char *dirname) {
 bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); }
 
 int32_t taosMkDir(const char *dirname) {
+  if (taosDirExist(dirname)) return 0;
+#ifdef WINDOWS
+  int32_t code = _mkdir(dirname, 0755);
+#else
   int32_t code = mkdir(dirname, 0755);
+#endif
   if (code < 0 && errno == EEXIST) {
     return 0;
   }
@@ -101,36 +106,48 @@ int32_t taosMkDir(const char *dirname) {
 
 int32_t taosMulMkDir(const char *dirname) {
   if (dirname == NULL) return -1;
-  char *  temp = strdup(dirname);
+  char temp[1024];
+#ifdef WINDOWS
+  taosRealPath(dirname, temp, sizeof(temp));
+#else
+  strcpy(temp, dirname);
+#endif
   char *  pos = temp;
   int32_t code = 0;
 
-  if (strncmp(temp, "/", 1) == 0) {
+  if (taosDirExist(temp)) return code;
+
+  if (strncmp(temp, TD_DIRSEP, 1) == 0) {
     pos += 1;
-  } else if (strncmp(temp, "./", 2) == 0) {
+  } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) {
     pos += 2;
   }
 
   for (; *pos != '\0'; pos++) {
-    if (*pos == '/') {
+    if (*pos == TD_DIRSEP[0]) {
       *pos = '\0';
+    #ifdef WINDOWS
+      code = _mkdir(temp, 0755);
+    #else
       code = mkdir(temp, 0755);
+    #endif
       if (code < 0 && errno != EEXIST) {
-        free(temp);
         return code;
       }
-      *pos = '/';
+      *pos = TD_DIRSEP[0];
     }
   }
 
-  if (*(pos - 1) != '/') {
+  if (*(pos - 1) != TD_DIRSEP[0]) {
+  #ifdef WINDOWS
+    code = _mkdir(temp, 0755);
+  #else
     code = mkdir(temp, 0755);
+  #endif
     if (code < 0 && errno != EEXIST) {
-      free(temp);
       return code;
     }
   }
-  free(temp);
 
   // int32_t code = mkdir(dirname, 0755);
   if (code < 0 && errno == EEXIST) {
@@ -233,8 +250,14 @@ char *taosDirName(char *name) {
   _splitpath(name, Drive1, Dir1, NULL, NULL);
   size_t dirNameLen = strlen(Drive1) + strlen(Dir1);
   if (dirNameLen > 0) {
+    if (name[dirNameLen - 1] == '/' || name[dirNameLen - 1] == '\\') {
+      name[dirNameLen - 1] = 0;
+    } else {
       name[dirNameLen] = 0;
     }
+  } else {
+    name[0] = 0;
+  }
   return name;
 #else
   return dirname(name);

source/os/src/osFile.c

@@ -109,8 +109,11 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha
 
 int64_t taosCopyFile(const char *from, const char *to) {
 #ifdef WINDOWS
-  assert(0);
+  if (CopyFile(from, to, 0)) {
+    return 1;
+  } else {
     return -1;
+  }
 #else
   char    buffer[4096];
   int64_t size = 0;
@@ -343,7 +346,11 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
   char   *tbuf = (char *)buf;
 
   while (leftbytes > 0) {
+  #ifdef WINDOWS
+    readbytes = _read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
+  #else
     readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes);
+  #endif
     if (readbytes < 0) {
       if (errno == EINTR) {
         continue;
@@ -379,10 +386,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
 #endif
   assert(pFile->fd >= 0);  // Please check if you have closed the file.
 #ifdef WINDOWS
-  size_t pos = lseek(pFile->fd, 0, SEEK_CUR);
+  size_t pos = _lseek(pFile->fd, 0, SEEK_CUR);
-  lseek(pFile->fd, offset, SEEK_SET);
+  _lseek(pFile->fd, offset, SEEK_SET);
-  int64_t ret = read(pFile->fd, buf, count);
+  int64_t ret = _read(pFile->fd, buf, count);
-  lseek(pFile->fd, pos, SEEK_SET);
+  _lseek(pFile->fd, pos, SEEK_SET);
 #else
   int64_t ret = pread(pFile->fd, buf, count, offset);
 #endif
@@ -428,7 +435,11 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
   taosThreadRwlockRdlock(&(pFile->rwlock));
 #endif
   assert(pFile->fd >= 0);  // Please check if you have closed the file.
+#ifdef WINDOWS
+  int64_t ret = _lseek(pFile->fd, offset, whence);
+#else
   int64_t ret = lseek(pFile->fd, offset, whence);
+#endif
 #if FILE_WITH_LOCK
   taosThreadRwlockUnlock(&(pFile->rwlock));
 #endif
@@ -567,12 +578,12 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
 
 #ifdef WINDOWS
 
-  lseek(pFileIn->fd, (int32_t)(*offset), 0);
+  _lseek(pFileIn->fd, (int32_t)(*offset), 0);
   int64_t writeLen = 0;
   uint8_t buffer[_SEND_FILE_STEP_] = {0};
 
   for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
-    size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_);
+    size_t rlen = _read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_);
     if (rlen <= 0) {
       return writeLen;
     } else if (rlen < _SEND_FILE_STEP_) {
@@ -586,7 +597,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in
 
   int64_t remain = size - writeLen;
   if (remain > 0) {
-    size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain);
+    size_t rlen = _read(pFileIn->fd, (void *)buffer, (size_t)remain);
     if (rlen <= 0) {
       return writeLen;
     } else {

source/os/src/osMemory.c

@@ -37,6 +37,49 @@ typedef struct TdMemoryInfo {
 
 #ifdef WINDOWS
 #define tstrdup(str) _strdup(str)
+
+int32_t taosBackTrace(void **buffer, int32_t size) {
+  int32_t frame = 0;
+  return frame;
+}
+
+#ifdef USE_ADDR2LINE
+#include <DbgHelp.h>
+#pragma comment(lib, "dbghelp.lib")
+
+void taosPrintBackTrace() {
+	#define MAX_STACK_FRAMES 20
+	
+	void *pStack[MAX_STACK_FRAMES];
+ 
+	HANDLE process = GetCurrentProcess();
+	SymInitialize(process, NULL, TRUE);
+	WORD frames = CaptureStackBackTrace(1, MAX_STACK_FRAMES, pStack, NULL);
+ 
+  char buf_tmp[1024];
+	for (WORD i = 0; i < frames; ++i) {
+		DWORD64 address = (DWORD64)(pStack[i]);
+ 
+		DWORD64 displacementSym = 0;
+		char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
+		PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
+		pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+		pSymbol->MaxNameLen = MAX_SYM_NAME;
+ 
+		DWORD displacementLine = 0;
+		IMAGEHLP_LINE64 line;
+		//SymSetOptions(SYMOPT_LOAD_LINES);
+		line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
+ 
+		if (SymFromAddr(process, address, &displacementSym, pSymbol) && SymGetLineFromAddr64(process, address, &displacementLine, &line)) {
+      snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace %08" PRId64 " %s:%d %s\n", taosGetSelfPthreadId(), line.FileName, line.LineNumber, pSymbol->Name);
+		} else {
+			snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace error: %d\n",GetLastError());
+		}
+    write(1,buf_tmp,strlen(buf_tmp));
+	}
+}
+#endif
 #else
 #define tstrdup(str) strdup(str)
 

source/os/src/osSemaphore.c

@@ -68,9 +68,32 @@ int32_t tsem_wait(tsem_t* sem) {
 }
 
 int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) {
-  int ret = 0;
+  struct timespec ts, rel;
+  FILETIME ft_before, ft_after;
+  int rc;
 
-  return ret;
+  rel.tv_sec = 0;
+  rel.tv_nsec = nanosecs;
+
+  GetSystemTimeAsFileTime(&ft_before);
+  errno = 0;
+  rc = sem_timedwait(&sem, pthread_win32_getabstime_np(&ts, &rel));
+
+  /* This should have timed out */
+  assert(errno == ETIMEDOUT);
+  assert(rc != 0);
+  GetSystemTimeAsFileTime(&ft_after);
+  // We specified a non-zero wait. Time must advance.
+  if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime)
+    {
+      printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n",
+          nanosecs, rc, errno,
+          (int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime,
+          (int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime);
+      printf("time must advance during sem_timedwait.");
+      return 1;
+    }
+  return 0;
 }
 
 #elif defined(_TD_DARWIN_64)

source/os/src/osSocket.c

@@ -718,7 +718,11 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) {
 
   bzero((char *)&serverAdd, sizeof(serverAdd));
   serverAdd.sin_family = AF_INET;
+#ifdef WINDOWS
+  serverAdd.sin_addr.s_addr = INADDR_ANY;
+#else
   serverAdd.sin_addr.s_addr = ip;
+#endif
   serverAdd.sin_port = (uint16_t)htons(port);
 
   if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) {
@@ -882,6 +886,16 @@ void taosBlockSIGPIPE() {
 }
 
 uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
+#ifdef WINDOWS
+  // Initialize Winsock
+  WSADATA wsaData;
+  int iResult;
+  iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (iResult != 0) {
+      printf("WSAStartup failed: %d\n", iResult);
+      return 1;
+  }
+#endif
   struct addrinfo hints = {0};
   hints.ai_family = AF_INET;
   hints.ai_socktype = SOCK_STREAM;
@@ -899,12 +913,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) {
   } else {
 #ifdef EAI_SYSTEM
     if (ret == EAI_SYSTEM) {
-      // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno));
+      printf("failed to get the ip address, fqdn:%s, errno:%d, since:%s", fqdn, errno, strerror(errno));
     } else {
-      // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
+      printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret));
     }
 #else
-    // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret));
+    printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret));
 #endif
     return 0xFFFFFFFF;
   }

source/util/src/tencode.c

@@ -30,7 +30,7 @@ struct SEncoderNode {
 
 struct SDecoderNode {
   SDecoderNode* pNext;
-  const uint8_t* data;
+  uint8_t*      data;
   uint32_t      size;
   uint32_t      pos;
 };
@@ -52,7 +52,7 @@ void tEncoderClear(SEncoder* pCoder) {
   memset(pCoder, 0, sizeof(*pCoder));
 }
 
-void tDecoderInit(SDecoder* pDecoder, const uint8_t* data, uint32_t size) {
+void tDecoderInit(SDecoder* pDecoder, uint8_t* data, uint32_t size) {
   pDecoder->data = data;
   pDecoder->size = size;
   pDecoder->pos = 0;

tools/taos-tools

@@ -1 +1 @@
-Subproject commit 772aef458fec13804c654e59c7a248b55ba3807b
+Subproject commit 2c4a1c83322b983881aea93ec2b51e7df826125a