diff --git a/cmake/cmake.define b/cmake/cmake.define index 1d34896f9a..0ae4f56f71 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -46,7 +46,7 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/w /D_WIN32") + SET(COMMON_FLAGS "/w /D_WIN32 /Zi") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") @@ -71,8 +71,8 @@ ELSE () ENDIF () IF (${SANITIZER} MATCHES "true") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -static-libasan -g3") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -g3") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=null -fno-sanitize=alignment -g3") MESSAGE(STATUS "Will compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index aba955ff3b..31b9936f3e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -100,8 +100,10 @@ endif(${BUILD_WITH_NURAFT}) # addr2line if(${BUILD_ADDR2LINE}) - cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + if(NOT ${TD_WINDOWS}) + cat("${TD_SUPPORT_DIR}/libdwarf_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${TD_SUPPORT_DIR}/addr2line_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + endif(NOT ${TD_WINDOWS}) endif(${BUILD_ADDR2LINE}) # download dependencies @@ -335,45 +337,47 @@ endif(${BUILD_WITH_SQLITE}) # addr2line if(${BUILD_ADDR2LINE}) - check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) - check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) - check_include_file( "inttypes.h" HAVE_INTTYPES_H ) - check_include_file( "stddef.h" HAVE_STDDEF_H ) - check_include_file( "stdlib.h" HAVE_STDLIB_H ) - check_include_file( "string.h" HAVE_STRING_H ) - check_include_file( "memory.h" HAVE_MEMORY_H ) - check_include_file( "strings.h" HAVE_STRINGS_H ) - check_include_file( "stdint.h" HAVE_STDINT_H ) - check_include_file( "unistd.h" HAVE_UNISTD_H ) - check_include_file( "sgidefs.h" HAVE_SGIDEFS_H ) - check_include_file( "stdafx.h" HAVE_STDAFX_H ) - check_include_file( "elf.h" HAVE_ELF_H ) - check_include_file( "libelf.h" HAVE_LIBELF_H ) - check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) - check_include_file( "alloca.h" HAVE_ALLOCA_H ) - check_include_file( "elfaccess.h" HAVE_ELFACCESS_H) - check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H ) - check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H) - check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H) - check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H ) - set(VERSION 0.3.1) - set(PACKAGE_VERSION "\"${VERSION}\"") - configure_file(libdwarf/cmake/config.h.cmake config.h) - file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c") - add_library(libdwarf STATIC ${LIBDWARF_SOURCES}) - set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf") - if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H) - target_link_libraries(libdwarf PUBLIC libelf) - endif() - target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR}) - file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT) - string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") - string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") - string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") - file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}") - add_library(addr2line STATIC "addr2line/addr2line.c") - target_link_libraries(addr2line PUBLIC libdwarf dl z) - target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" ) + if(NOT ${TD_WINDOWS}) + check_include_file( "sys/types.h" HAVE_SYS_TYPES_H) + check_include_file( "sys/stat.h" HAVE_SYS_STAT_H ) + check_include_file( "inttypes.h" HAVE_INTTYPES_H ) + check_include_file( "stddef.h" HAVE_STDDEF_H ) + check_include_file( "stdlib.h" HAVE_STDLIB_H ) + check_include_file( "string.h" HAVE_STRING_H ) + check_include_file( "memory.h" HAVE_MEMORY_H ) + check_include_file( "strings.h" HAVE_STRINGS_H ) + check_include_file( "stdint.h" HAVE_STDINT_H ) + check_include_file( "unistd.h" HAVE_UNISTD_H ) + check_include_file( "sgidefs.h" HAVE_SGIDEFS_H ) + check_include_file( "stdafx.h" HAVE_STDAFX_H ) + check_include_file( "elf.h" HAVE_ELF_H ) + check_include_file( "libelf.h" HAVE_LIBELF_H ) + check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) + check_include_file( "alloca.h" HAVE_ALLOCA_H ) + check_include_file( "elfaccess.h" HAVE_ELFACCESS_H) + check_include_file( "sys/elf_386.h" HAVE_SYS_ELF_386_H ) + check_include_file( "sys/elf_amd64.h" HAVE_SYS_ELF_AMD64_H) + check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H) + check_include_file( "sys/ia64/elf.h" HAVE_SYS_IA64_ELF_H ) + set(VERSION 0.3.1) + set(PACKAGE_VERSION "\"${VERSION}\"") + configure_file(libdwarf/cmake/config.h.cmake config.h) + file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c") + add_library(libdwarf STATIC ${LIBDWARF_SOURCES}) + set_target_properties(libdwarf PROPERTIES OUTPUT_NAME "libdwarf") + if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H) + target_link_libraries(libdwarf PUBLIC libelf) + endif() + target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR}) + file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT) + string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") + string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") + string(REPLACE "main(" "main_addr2line(" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}") + file(WRITE "addr2line/addr2line.c" "${ADDR2LINE_CONTENT}") + add_library(addr2line STATIC "addr2line/addr2line.c") + target_link_libraries(addr2line PUBLIC libdwarf dl z) + target_include_directories(addr2line PUBLIC "libdwarf/src/lib/libdwarf" ) + endif(NOT ${TD_WINDOWS}) endif(${BUILD_ADDR2LINE}) diff --git a/docs-cn/14-reference/03-connector/csharp.mdx b/docs-cn/14-reference/03-connector/csharp.mdx index c2fbb3b67f..1e23df9286 100644 --- a/docs-cn/14-reference/03-connector/csharp.mdx +++ b/docs-cn/14-reference/03-connector/csharp.mdx @@ -18,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。 -`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式,用户可以参考 [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) 文档自行编写。 +`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、订阅、schemaless 数据写入、参数绑定接口数据写入等功能 `TDengine.Connector` 目前暂未提供 REST 连接方式,用户可以参考 [REST API](/reference/rest-api/) 文档自行编写。 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`,并通过 `TDengine.Connector` 连接 TDengine 集群,进行数据写入、查询等基本操作。 diff --git a/docs-en/14-reference/03-connector/csharp.mdx b/docs-en/14-reference/03-connector/csharp.mdx index ca4b1b9ece..2969392a05 100644 --- a/docs-en/14-reference/03-connector/csharp.mdx +++ b/docs-en/14-reference/03-connector/csharp.mdx @@ -19,7 +19,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data. -The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [RESTful APIs](https://docs.taosdata.com//reference/restful-api/) documentation. +The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc. The `TDengine.Connector` currently does not provide a REST connection interface. Developers can write their RESTful application by referring to the [REST API](/reference/rest-api/) documentation. This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying. diff --git a/example/src/tstream.c b/example/src/tstream.c index 537bfebede..97ff2886fc 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -82,9 +82,7 @@ int32_t create_stream() { /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ pRes = taos_query( - pConn, - "create stream stream1 trigger at_once into outstb as select _wstartts, min(k), max(k), sum(k) as sum_of_k " - "from tu1 interval(10m)"); + pConn, "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b893099064..acf2587d9d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -244,12 +244,12 @@ typedef struct { const void* pMsg; } SSubmitMsgIter; -int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); +int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); // for debug -int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); +int32_t tPrintFixedSchemaSubmitReq(SSubmitReq* pReq, STSchema* pSchema); typedef struct { int32_t code; @@ -1697,7 +1697,7 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam); // TDMT_VND_CREATE_STB ============== typedef struct SVCreateStbReq { - const char* name; + char* name; tb_uid_t suid; int8_t rollup; SSchemaWrapper schema; @@ -1710,8 +1710,8 @@ int tDecodeSVCreateStbReq(SDecoder* pCoder, SVCreateStbReq* pReq); // TDMT_VND_DROP_STB ============== typedef struct SVDropStbReq { - const char* name; - tb_uid_t suid; + char* name; + tb_uid_t suid; } SVDropStbReq; int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq); @@ -1720,16 +1720,16 @@ int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq); // TDMT_VND_CREATE_TABLE ============== #define TD_CREATE_IF_NOT_EXISTS 0x1 typedef struct SVCreateTbReq { - int32_t flags; - tb_uid_t uid; - int64_t ctime; - const char* name; - int32_t ttl; - int8_t type; + int32_t flags; + tb_uid_t uid; + int64_t ctime; + char* name; + int32_t ttl; + int8_t type; union { struct { - tb_uid_t suid; - const uint8_t* pTag; + tb_uid_t suid; + uint8_t* pTag; } ctb; struct { SSchemaWrapper schema; @@ -1777,8 +1777,8 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc // TDMT_VND_DROP_TABLE ================= typedef struct { - const char* name; - int8_t igNotExists; + char* name; + int8_t igNotExists; } SVDropTbReq; typedef struct { @@ -1809,9 +1809,9 @@ int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp); // TDMT_VND_ALTER_TABLE ===================== typedef struct { - const char* tbName; - int8_t action; - const char* colName; + char* tbName; + int8_t action; + char* colName; // TSDB_ALTER_TABLE_ADD_COLUMN int8_t type; int8_t flags; @@ -1820,17 +1820,17 @@ typedef struct { // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES int32_t colModBytes; // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME - const char* colNewName; + char* colNewName; // TSDB_ALTER_TABLE_UPDATE_TAG_VAL - const char* tagName; - int8_t isNull; - uint32_t nTagVal; - const uint8_t* pTagVal; + char* tagName; + int8_t isNull; + uint32_t nTagVal; + uint8_t* pTagVal; // TSDB_ALTER_TABLE_UPDATE_OPTIONS - int8_t updateTTL; - int32_t newTTL; - int8_t updateComment; - const char* newComment; + int8_t updateTTL; + int32_t newTTL; + int8_t updateComment; + char* newComment; } SVAlterTbReq; int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); @@ -2020,7 +2020,7 @@ static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); -void tFreeSClientHbBatchRsp(SClientHbBatchRsp *pBatchRsp); +void tFreeSClientHbBatchRsp(SClientHbBatchRsp* pBatchRsp); static FORCE_INLINE int32_t tEncodeSKv(SEncoder* pEncoder, const SKv* pKv) { if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; @@ -2255,20 +2255,20 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - const char* expr; // sma expression - const char* tagsFilter; + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* expr; // sma expression + char* tagsFilter; } STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2596,12 +2596,12 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { #define TD_AUTO_CREATE_TABLE 0x1 typedef struct { - int64_t suid; - int64_t uid; - int32_t sver; - uint32_t nData; - const uint8_t* pData; - SVCreateTbReq cTbReq; + int64_t suid; + int64_t uid; + int32_t sver; + uint32_t nData; + uint8_t* pData; + SVCreateTbReq cTbReq; } SVSubmitBlk; typedef struct { diff --git a/include/libs/index/index.h b/include/libs/index/index.h index fa4cb1d2bd..05db99db0f 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -16,9 +16,11 @@ #ifndef _TD_INDEX_H_ #define _TD_INDEX_H_ +#include "nodes.h" #include "os.h" #include "taoserror.h" #include "tarray.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { @@ -189,6 +191,12 @@ void indexTermDestroy(SIndexTerm* p); */ void indexInit(); +/* index filter */ +typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; + +SIdxFltStatus idxGetFltStatus(SNode* pFilterNode); + +int32_t doFilterTag(const SNode* pFilterNode, SArray* result); /* * destory index env * diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 291e08fdbf..b9cb708c9c 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -59,10 +59,10 @@ extern "C" { for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) -#define DESTORY_LIST(list) \ - do { \ - nodesDestroyList(list); \ - list = NULL; \ +#define DESTORY_LIST(list) \ + do { \ + nodesDestroyList((list)); \ + (list) = NULL; \ } while (0) typedef enum ENodeType { @@ -96,6 +96,7 @@ typedef enum ENodeType { QUERY_NODE_EXPLAIN_OPTIONS, QUERY_NODE_STREAM_OPTIONS, QUERY_NODE_TOPIC_OPTIONS, + QUERY_NODE_LEFT_VALUE, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 298dffcc83..716dfc17dc 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -81,6 +81,7 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; + bool notReserved; int16_t placeholderNo; union { bool b; @@ -93,6 +94,10 @@ typedef struct SValueNode { char unit; } SValueNode; +typedef struct SLeftValueNode { + ENodeType type; +} SLeftValueNode; + typedef struct SOperatorNode { SExprNode node; // QUERY_NODE_OPERATOR EOperatorType opType; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1604749af8..d18f609d54 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -114,17 +114,12 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput); void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput); -typedef struct { - void* inputHandle; - void* executor; -} SStreamRunner; - typedef struct { int8_t parallelizable; char* qmsg; // followings are not applicable to encoder and decoder - int8_t numOfRunners; - SStreamRunner* runners; + void* inputHandle; + void* executor; } STaskExec; typedef struct { @@ -320,17 +315,15 @@ int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input); int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input); int32_t streamDequeueOutput(SStreamTask* pTask, void** output); -int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId); - int32_t streamTaskRun(SStreamTask* pTask); int32_t streamTaskHandleInput(SStreamTask* pTask, void* data); int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb); -int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg); -int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp); -int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); -int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); +int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pMsg); +int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp); +int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); +int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index 4168a24003..808fcf0152 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -132,6 +132,7 @@ typedef enum EOperatorType { OP_TYPE_MOD, // unary arithmetic operator OP_TYPE_MINUS, + OP_TYPE_ASSIGN, // bit operator OP_TYPE_BIT_AND, diff --git a/include/util/tencode.h b/include/util/tencode.h index 938e3018a8..af38d694e2 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -39,11 +39,11 @@ typedef struct { } SEncoder; typedef struct { - const uint8_t* data; - uint32_t size; - uint32_t pos; - SCoderMem* mList; - SDecoderNode* dStack; + uint8_t* data; + uint32_t size; + uint32_t pos; + SCoderMem* mList; + SDecoderNode* dStack; } SDecoder; #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) @@ -120,7 +120,7 @@ static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t le static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); /* ------------------------ DECODE ------------------------ */ -void tDecoderInit(SDecoder* pCoder, const uint8_t* data, uint32_t size); +void tDecoderInit(SDecoder* pCoder, uint8_t* data, uint32_t size); void tDecoderClear(SDecoder* SDecoder); int32_t tStartDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder); @@ -141,9 +141,9 @@ static int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val); static int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val); static int32_t tDecodeFloat(SDecoder* pCoder, float* val); static int32_t tDecodeDouble(SDecoder* pCoder, double* val); -static int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len); -static int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len); -static int32_t tDecodeCStr(SDecoder* pCoder, const char** val); +static int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len); +static int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len); +static int32_t tDecodeCStr(SDecoder* pCoder, char** val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); /* ------------------------ IMPL ------------------------ */ @@ -317,7 +317,7 @@ static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { if (tDecodeU16v(pCoder, &tval) < 0) { return -1; } - *val = ZIGZAGD(int16_t, tval); + if (val) *val = ZIGZAGD(int16_t, tval); return 0; } @@ -331,7 +331,7 @@ static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { if (tDecodeU32v(pCoder, &tval) < 0) { return -1; } - *val = ZIGZAGD(int32_t, tval); + if (val) *val = ZIGZAGD(int32_t, tval); return 0; } @@ -345,7 +345,7 @@ static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { if (tDecodeU64v(pCoder, &tval) < 0) { return -1; } - *val = ZIGZAGD(int64_t, tval); + if (val) *val = ZIGZAGD(int64_t, tval); return 0; } @@ -377,7 +377,7 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) { return 0; } -static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, uint32_t* len) { +static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { if (tDecodeU32v(pCoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, *len)) return -1; @@ -389,20 +389,20 @@ static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, const uint8_t** val, return 0; } -static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, const char** val, uint32_t* len) { - if (tDecodeBinary(pCoder, (const uint8_t**)val, len) < 0) return -1; +static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { + if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1; (*len) -= 1; return 0; } -static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, const char** val) { +static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) { uint32_t len; return tDecodeCStrAndLen(pCoder, val, &len); } static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { - const char* pStr; - uint32_t len; + char* pStr; + uint32_t len; if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; memcpy(val, pStr, len + 1); diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 2b26674324..f07705ff44 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -6,16 +6,36 @@ set -e #set -x +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" + # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory -data_dir="/var/lib/taos" -log_dir="/var/log/taos" -data_link_dir="/usr/local/taos/data" -log_link_dir="/usr/local/taos/log" +clientName="taos" +serverName="taosd" +configFile="taos.cfg" +productName="TDengine" +emailName="taosdata.com" +uninstallScript="rmtaos" +historyFile="taos_history" +tarName="taos.tar.gz" +dataDir="/var/lib/taos" +logDir="/var/log/taos" +configDir="/etc/taos" +installDir="/usr/local/taos" +adapterName="taosadapter" +benchmarkName="taosBenchmark" +dumpName="taosdump" +demoName="taosdemo" -cfg_install_dir="/etc/taos" +data_dir=${dataDir} +log_dir=${logDir} +cfg_install_dir=${configDir} bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" @@ -23,21 +43,13 @@ lib64_link_dir="/usr/lib64" inc_link_dir="/usr/include" #install main path -install_main_dir="/usr/local/taos" - +install_main_dir=${installDir} # old bin dir -bin_dir="/usr/local/taos/bin" +bin_dir="${installDir}/bin" service_config_dir="/etc/systemd/system" - -#taos-tools para -demoName="taosdemo" -benchmarkName="taosBenchmark" -dumpName="taosdump" -emailName="taosdata.com" -taosName="taos" -toolsName="taostools" - +nginx_port=6060 +nginx_dir="/usr/local/nginxd" # Color setting RED='\033[0;31m' @@ -47,8 +59,8 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" +if command -v sudo >/dev/null; then + csudo="sudo " fi update_flag=0 @@ -56,52 +68,51 @@ prompt_force=0 initd_mod=0 service_mod=2 -if pidof systemd &> /dev/null; then - service_mod=0 -elif $(which service &> /dev/null); then - service_mod=1 - service_config_dir="/etc/init.d" - if $(which chkconfig &> /dev/null); then - initd_mod=1 - elif $(which insserv &> /dev/null); then - initd_mod=2 - elif $(which update-rc.d &> /dev/null); then - initd_mod=3 - else - service_mod=2 - fi -else +if pidof systemd &>/dev/null; then + service_mod=0 +elif $(which service &>/dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &>/dev/null); then + initd_mod=1 + elif $(which insserv &>/dev/null); then + initd_mod=2 + elif $(which update-rc.d &>/dev/null); then + initd_mod=3 + else service_mod=2 + fi +else + service_mod=2 fi - # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) if [[ -e /etc/os-release ]]; then - osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || : else osinfo="" fi #echo "osinfo: ${osinfo}" os_type=0 -if echo $osinfo | grep -qwi "ubuntu" ; then -# echo "This is ubuntu system" +if echo $osinfo | grep -qwi "ubuntu"; then + # echo "This is ubuntu system" os_type=1 -elif echo $osinfo | grep -qwi "debian" ; then -# echo "This is debian system" +elif echo $osinfo | grep -qwi "debian"; then + # echo "This is debian system" os_type=1 -elif echo $osinfo | grep -qwi "Kylin" ; then -# echo "This is Kylin system" +elif echo $osinfo | grep -qwi "Kylin"; then + # echo "This is Kylin system" os_type=1 -elif echo $osinfo | grep -qwi "centos" ; then -# echo "This is centos system" +elif echo $osinfo | grep -qwi "centos"; then + # echo "This is centos system" os_type=2 -elif echo $osinfo | grep -qwi "fedora" ; then -# echo "This is fedora system" +elif echo $osinfo | grep -qwi "fedora"; then + # echo "This is fedora system" os_type=2 -elif echo $osinfo | grep -qwi "Linx" ; then -# echo "This is Linx system" +elif echo $osinfo | grep -qwi "Linx"; then + # echo "This is Linx system" os_type=1 service_mod=0 initd_mod=0 @@ -110,43 +121,41 @@ else echo " osinfo: ${osinfo}" echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " - echo " please feel free to contact taosdata.com for support." + echo " please feel free to contact ${emailName} for support." os_type=1 fi - # ============================= get input parameters ================================================= # install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] # set parameters by default value -interactiveFqdn=yes # [yes | no] -verType=server # [server | client] -initType=systemd # [systemd | service | ...] +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] -while getopts "hv:e:i:" arg -do +while getopts "hv:e:i:" arg; do case $arg in - e) - #echo "interactiveFqdn=$OPTARG" - interactiveFqdn=$( echo $OPTARG ) - ;; - v) - #echo "verType=$OPTARG" - verType=$(echo $OPTARG) - ;; - i) - #echo "initType=$OPTARG" - initType=$(echo $OPTARG) - ;; - h) - echo "Usage: `basename $0` -v [server | client] -e [yes | no]" - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$(echo $OPTARG) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: $(basename $0) -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; esac done @@ -155,98 +164,163 @@ done function kill_process() { pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then - ${csudo} kill -9 $pid || : + ${csudo}kill -9 $pid || : fi } function install_main_path() { - #create install main dir and all sub dir - ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin - ${csudo} mkdir -p ${install_main_dir}/connector - ${csudo} mkdir -p ${install_main_dir}/lib - ${csudo} mkdir -p ${install_main_dir}/examples - ${csudo} mkdir -p ${install_main_dir}/include - ${csudo} mkdir -p ${install_main_dir}/init.d - if [ "$verMode" == "cluster" ]; then - ${csudo} mkdir -p ${nginx_dir} - fi + #create install main dir and all sub dir + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/cfg + ${csudo}mkdir -p ${install_main_dir}/bin + # ${csudo}mkdir -p ${install_main_dir}/connector + ${csudo}mkdir -p ${install_main_dir}/driver + ${csudo}mkdir -p ${install_main_dir}/examples + ${csudo}mkdir -p ${install_main_dir}/include + # ${csudo}mkdir -p ${install_main_dir}/init.d + if [ "$verMode" == "cluster" ]; then + ${csudo}mkdir -p ${nginx_dir} + fi - if [[ -e ${script_dir}/email ]]; then - ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: - fi + if [[ -e ${script_dir}/email ]]; then + ${csudo}cp ${script_dir}/email ${install_main_dir}/ || : + fi } function install_bin() { - # Remove links - ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosd || : - ${csudo} rm -f ${bin_link_dir}/taosadapter || : - ${csudo} rm -f ${bin_link_dir}/create_table || : - ${csudo} rm -f ${bin_link_dir}/tmq_sim || : - ${csudo} rm -f ${bin_link_dir}/taosdump || : - ${csudo} rm -f ${bin_link_dir}/rmtaos || : - #${csudo} rm -f ${bin_link_dir}/set_core || : + # Remove links + ${csudo}rm -f ${bin_link_dir}/${clientName} || : + ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${adapterName} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/tarbitrator || : + ${csudo}rm -f ${bin_link_dir}/set_core || : + ${csudo}rm -f ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || : + ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* - #Make link - [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : - [ -x ${install_main_dir}/bin/create_table ] && ${csudo} ln -s ${install_main_dir}/bin/create_table ${bin_link_dir}/create_table || : - [ -x ${install_main_dir}/bin/tmq_sim ] && ${csudo} ln -s ${install_main_dir}/bin/tmq_sim ${bin_link_dir}/tmq_sim || : -# [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : -# [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : -# [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + #Make link + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : + [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : + [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : + [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : + [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ] && ${csudo}ln -s ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : + + if [ "$verMode" == "cluster" ]; then + ${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/* + ${csudo}mkdir -p ${nginx_dir}/logs + ${csudo}chmod 777 ${nginx_dir}/sbin/nginx + fi } function install_lib() { - # Remove links - ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib_link_dir}/libtdb.* || : - ${csudo} rm -f ${lib64_link_dir}/libtdb.* || : + # Remove links + ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo}rm -rf ${v15_java_app_dir} || : + ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* - ${csudo} cp -rf ${script_dir}/lib/* ${install_main_dir}/lib && ${csudo} chmod 777 ${install_main_dir}/lib/* + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi - if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then - ${csudo} ln -s ${install_main_dir}/lib/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ldconfig +} + +function install_avro() { + if [ "$osType" != "Darwin" ]; then + avro_dir=${script_dir}/avro + if [ -f "${avro_dir}/lib/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/$1 + ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.so.23.0.0 /usr/local/$1 + ${csudo}ln -sf /usr/local/$1/libavro.so.23.0.0 /usr/local/$1/libavro.so.23 + ${csudo}ln -sf /usr/local/$1/libavro.so.23 /usr/local/$1/libavro.so + + ${csudo}/usr/bin/install -c -d /usr/local/$1 + [ -f ${avro_dir}/lib/libavro.a ] && + ${csudo}/usr/bin/install -c -m 755 ${avro_dir}/lib/libavro.a /usr/local/$1 + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/$1" | ${csudo}tee /etc/ld.so.conf.d/libavro.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/libavro.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi + fi +} + +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 fi - ${csudo} ldconfig + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi } function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* - ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h -# ${csudo} ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h -} - -# temp install taosBenchmark -function install_taosTools() { - ${csudo} rm -f ${bin_link_dir}/${benchmarkName} || : - ${csudo} rm -f ${bin_link_dir}/${dumpName} || : - ${csudo} rm -f ${bin_link_dir}/rm${toolsName} || : - - ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName} - ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName} - ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName} - #Make link - [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \ - ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [[ -x ${install_main_dir}/bin/${demoName} ]] && \ - ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName} || : - [[ -x ${install_main_dir}/bin/${dumpName} ]] && \ - ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function add_newHostname_to_hosts() { @@ -256,18 +330,17 @@ function add_newHostname_to_hosts() { iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') arr=($iphost) IFS="$OLD_IFS" - for s in "${arr[@]}" - do + for s in "${arr[@]}"; do if [[ "$s" == "$localIp" ]]; then return fi done - ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: + ${csudo}echo "127.0.0.1 $1" >>/etc/hosts || : } function set_hostname() { echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + read newHostname while true; do if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then break @@ -276,28 +349,25 @@ function set_hostname() { fi done - ${csudo} hostname $newHostname ||: - retval=`echo $?` + ${csudo}hostname $newHostname || : + retval=$(echo $?) if [[ $retval != 0 ]]; then - echo - echo "set hostname fail!" - return + echo + echo "set hostname fail!" + return fi - #echo -e -n "$(hostnamectl status --static)" - #echo -e -n "$(hostnamectl status --transient)" - #echo -e -n "$(hostnamectl status --pretty)" #ubuntu/centos /etc/hostname if [[ -e /etc/hostname ]]; then - ${csudo} echo $newHostname > /etc/hostname ||: + ${csudo}echo $newHostname >/etc/hostname || : fi #debian: #HOSTNAME=yourname if [[ -e /etc/sysconfig/network ]]; then - ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : fi - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile} serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -311,20 +381,19 @@ function is_correct_ipaddr() { IFS=" " arr=($iplist) IFS="$OLD_IFS" - for s in "${arr[@]}" - do - if [[ "$s" == "$newIp" ]]; then - return 0 - fi + for s in "${arr[@]}"; do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi done return 1 } function set_ipAsFqdn() { - iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + iplist=$(ip address | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F "/" '{print $1}') || : if [ -z "$iplist" ]; then - iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + iplist=$(ifconfig | grep inet | grep -v inet6 | grep -v 127.0.0.1 | awk '{print $2}' | awk -F ":" '{print $2}') || : fi if [ -z "$iplist" ]; then @@ -332,7 +401,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} serverFqdn=$localFqdn echo return @@ -345,23 +414,23 @@ function set_ipAsFqdn() { echo echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" read localFqdn - while true; do - if [ ! -z "$localFqdn" ]; then - # Check if correct ip address - is_correct_ipaddr $localFqdn - retval=`echo $?` - if [[ $retval != 0 ]]; then - read -p "Please choose an IP from local IP list:" localFqdn - else - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg - serverFqdn=$localFqdn - break - fi - else + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=$(echo $?) + if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + serverFqdn=$localFqdn + break fi - done + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done } function local_fqdn_check() { @@ -369,205 +438,553 @@ function local_fqdn_check() { echo echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" echo - while true - do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS]|[yY]) - set_hostname - break - ;; + while true; do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS] | [yY]) + set_hostname + break + ;; - [nN][oO]|[nN]) - set_ipAsFqdn - break - ;; + [nN][oO] | [nN]) + set_ipAsFqdn + break + ;; - *) - echo "Invalid input..." - ;; - esac - fi + *) + echo "Invalid input..." + ;; + esac + fi done fi } -function install_log() { - ${csudo} rm -rf ${log_dir} || : - ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} +function install_adapter_config() { + if [ ! -f "${cfg_install_dir}/${adapterName}.toml" ]; then + ${csudo}mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/${adapterName}.toml + fi - ${csudo} ln -s ${log_dir} ${install_main_dir}/log -} + [ -f ${script_dir}/cfg/${adapterName}.toml ] && + ${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new -function install_data() { - ${csudo} mkdir -p ${data_dir} + [ -f ${cfg_install_dir}/${adapterName}.toml ] && + ${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml - ${csudo} ln -s ${data_dir} ${install_main_dir}/data -} + [ ! -z $1 ] && return 0 || : # only install client -function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/taosd.service" - if systemctl is-active --quiet taosd; then - echo "TDengine is running, stopping it..." - ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${taosd_service_config} - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${tarbitratord_service_config} - - if [ "$verMode" == "cluster" ]; then - nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then - echo "Nginx for TDengine is running, stopping it..." - ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi -} - -# taos:2345:respawn:/etc/init.d/taosd start - -function install_service_on_systemd() { - clean_service_on_systemd - - taosd_service_config="${service_config_dir}/taosd.service" - ${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'After=network-online.target taosadapter.service' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target taosadapter.service' >> ${taosd_service_config}" - ${csudo} bash -c "echo >> ${taosd_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" - #${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}" - ${csudo} bash -c "echo >> ${taosd_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" - ${csudo} systemctl enable taosd - - ${csudo} systemctl daemon-reload -} - -function install_service() { - # if ((${service_mod}==0)); then - # install_service_on_systemd - # elif ((${service_mod}==1)); then - # install_service_on_sysvinit - # else - # # must manual stop taosd - kill_process taosd - # fi } function install_config() { - if [ ! -f ${cfg_install_dir}/${configFile} ]; then - ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} - ${csudo}chmod 644 ${cfg_install_dir}/* - fi - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${install_main_dir}/cfg/${configFile}.org - ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + if [ ! -f "${cfg_install_dir}/${configFile}" ]; then + ${csudo}mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + ${csudo}chmod 644 ${cfg_install_dir}/* + fi + + ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new + ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag} == 1)); then + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi + + local_fqdn_check + + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile} + break + else + break + fi + done + + echo + echo -e -n "${GREEN}Enter your email address for priority support or enter empty to skip${NC}: " + read emailAddr + while true; do + if [ ! -z "$emailAddr" ]; then + email_file="${install_main_dir}/email" + ${csudo}bash -c "echo $emailAddr > ${email_file}" + break + else + break + fi + done } -function install_TDengine() { - # Start to install - echo -e "${GREEN}Start to install TDengine...${NC}" +function install_log() { + ${csudo}rm -rf ${log_dir} || : + ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - install_main_path - install_data - install_log - install_header - install_lib - install_taosTools + ${csudo}ln -s ${log_dir} ${install_main_dir}/log +} - if [ -z $1 ]; then # install service and client - # For installing new - install_bin - install_service - install_config +function install_data() { + ${csudo}mkdir -p ${data_dir} - # Ask if to start the service - #echo - #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" - echo - echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" - if ((${service_mod}==0)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" - elif ((${service_mod}==1)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" + ${csudo}ln -s ${data_dir} ${install_main_dir}/data +} + +function install_connector() { + [ -d "${script_dir}/connector/" ] && ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function clean_service_on_sysvinit() { + if pidof ${serverName} &>/dev/null; then + ${csudo}service ${serverName} stop || : + fi + + if pidof tarbitrator &>/dev/null; then + ${csudo}service tarbitratord stop || : + fi + + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/${serverName} ]; then + ${csudo}chkconfig --del ${serverName} || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}chkconfig --del tarbitratord || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/${serverName} ]; then + ${csudo}insserv -r ${serverName} || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}insserv -r tarbitratord || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/${serverName} ]; then + ${csudo}update-rc.d -f ${serverName} remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo}update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/${serverName} || : + ${csudo}rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + if ((${os_type} == 1)); then + # ${csudo}cp -f ${script_dir}/init.d/${serverName}.deb ${install_main_dir}/init.d/${serverName} + ${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} + # ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type} == 2)); then + # ${csudo}cp -f ${script_dir}/init.d/${serverName}.rpm ${install_main_dir}/init.d/${serverName} + ${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} + # ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord + fi + + if ((${initd_mod} == 1)); then + ${csudo}chkconfig --add ${serverName} || : + ${csudo}chkconfig --level 2345 ${serverName} on || : + ${csudo}chkconfig --add tarbitratord || : + ${csudo}chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod} == 2)); then + ${csudo}insserv ${serverName} || : + ${csudo}insserv -d ${serverName} || : + ${csudo}insserv tarbitratord || : + ${csudo}insserv -d tarbitratord || : + elif ((${initd_mod} == 3)); then + ${csudo}update-rc.d ${serverName} defaults || : + ${csudo}update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + taosd_service_config="${service_config_dir}/${serverName}.service" + if systemctl is-active --quiet ${serverName}; then + echo "${productName} is running, stopping it..." + ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${taosd_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for ${productName} is running, stopping it..." + ${csudo}systemctl stop nginxd &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable nginxd &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${nginx_service_config} + fi +} + +function install_service_on_systemd() { + clean_service_on_systemd + + [ -f ${script_dir}/cfg/${serverName}.service ] && + ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + + ${csudo}systemctl enable ${serverName} + + [ -f ${script_dir}/cfg/tarbitratord.service ] && + ${csudo}cp ${script_dir}/cfg/tarbitratord.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + + if [ "$verMode" == "cluster" ]; then + [ -f ${script_dir}/cfg/nginxd.service ] && + ${csudo}cp ${script_dir}/cfg/nginxd.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + + if ! ${csudo}systemctl enable nginxd &>/dev/null; then + ${csudo}systemctl daemon-reexec + ${csudo}systemctl enable nginxd + fi + ${csudo}systemctl start nginxd + fi +} + +function install_adapter_service() { + if ((${service_mod} == 0)); then + [ -f ${script_dir}/cfg/${adapterName}.service ] && + ${csudo}cp ${script_dir}/cfg/${adapterName}.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + +function install_service() { + if ((${service_mod} == 0)); then + install_service_on_systemd + elif ((${service_mod} == 1)); then + install_service_on_sysvinit + else + kill_process ${serverName} + fi +} + +vercomp() { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i = 0; i < ${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + + curr_version=$(ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}') + + if [ -f ${script_dir}/driver/vercomp.txt ]; then + min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) + else + min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + fi + + exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + vercomp $exist_version "2.0.16.0" + case $? in + 2) + prompt_force=1 + ;; + esac + + vercomp $curr_version $min_compatible_version + echo "" # avoid $? value not update + + case $? in + 0) return 0 ;; + 1) return 0 ;; + 2) return 1 ;; + esac +} + +function updateProduct() { + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + # Start to update + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + install_jemalloc + + echo -e "${GREEN}Start to update ${productName}...${NC}" + # Stop the service if running + if pidof ${serverName} &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop ${serverName} || : + elif ((${service_mod} == 1)); then + ${csudo}service ${serverName} stop || : + else + kill_process ${serverName} + fi + sleep 1 + fi + + if [ "$verMode" == "cluster" ]; then + if pidof nginx &>/dev/null; then + if ((${service_mod} == 0)); then + ${csudo}systemctl stop nginxd || : + elif ((${service_mod} == 1)); then + ${csudo}service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi + fi + + install_main_path + + install_log + install_header + install_lib + + if [ "$verMode" == "cluster" ]; then + install_connector + fi + + install_examples + if [ -z $1 ]; then + install_bin + install_service + install_adapter_service + install_config + install_adapter_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if openresty is installed + # Check if nginx is installed successfully + if type curl &>/dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then + echo -e "\033[44;32;1mNginx for ${productName} is updated successfully!${NC}" + openresty_work=true else - echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" + echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m" fi - - if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo - elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" - echo - fi - - echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" - echo - else # Only install client - install_bin - install_config - echo - echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" + fi fi - touch ~/.taos_history + echo + echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" + echo -e "${GREEN_DARK}To configure Adapter (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml" + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + else + echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}" + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" + fi + + if [ ${openresty_work} = 'true' ]; then + echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + else + echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}" + fi + + if ((${prompt_force} == 1)); then + echo "" + echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}" + fi + echo + echo -e "\033[44;32;1m${productName} is updated successfully!${NC}" + else + install_bin + install_config + + echo + echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}" + fi + + rm -rf $(tar -tf ${tarName} | grep -v "^\./$") } +function installProduct() { + # Start to install + if [ ! -e ${tarName} ]; then + echo "File ${tarName} does not exist" + exit 1 + fi + tar -zxf ${tarName} + + echo -e "${GREEN}Start to install ${productName}...${NC}" + + install_main_path + + if [ -z $1 ]; then + install_data + fi + + install_log + install_header + install_lib + install_jemalloc + #install_avro lib + #install_avro lib64 + + if [ "$verMode" == "cluster" ]; then + install_connector + fi + install_examples + + if [ -z $1 ]; then # install service and client + # For installing new + install_bin + install_service + install_adapter_service + install_adapter_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if nginx is installed successfully + if type curl &>/dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &>/dev/null; then + echo -e "\033[44;32;1mNginx for ${productName} is installed successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for ${productName} does not work! Please try again!\033[0m" + fi + fi + fi + + install_config + + # Ask if to start the service + echo + echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" + echo -e "${GREEN_DARK}To configure ${adapterName} (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml" + if ((${service_mod} == 0)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + elif ((${service_mod} == 1)); then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + else + echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}" + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + fi + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]]; then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]]; then + echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $serverFqdn${GREEN_DARK} to login into ${productName} server${NC}" + echo + fi + + echo -e "\033[44;32;1m${productName} is installed successfully!${NC}" + echo + else # Only install client + install_bin + install_config + echo + echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" + fi + + touch ~/.${historyFile} + rm -rf $(tar -tf ${tarName} | grep -v "^\./$") +} ## ==============================Main program starts from here============================ serverFqdn=$(hostname) if [ "$verType" == "server" ]; then - # Install server and client - install_TDengine + # Install server and client + if [ -x ${bin_dir}/${serverName} ]; then + update_flag=1 + updateProduct + else + installProduct + fi elif [ "$verType" == "client" ]; then - interactiveFqdn=no - # Only install client - install_TDengine client + interactiveFqdn=no + # Only install client + if [ -x ${bin_dir}/${clientName} ]; then + update_flag=1 + updateProduct client + else + installProduct client + fi else - echo "please input correct verType" + echo "please input correct verType" fi diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cc333ae5c8..b381299d27 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -28,7 +28,7 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -102,7 +102,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } -int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) { +int32_t tPrintFixedSchemaSubmitReq(SSubmitReq *pReq, STSchema *pTschema) { SSubmitMsgIter msgIter = {0}; if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; while (true) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index e7bfbdae58..6183794bdd 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -113,6 +113,8 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); SRpcMsg rsp = {.info = pMsg->info}; + vnodePreprocessReq(pVnode->pImpl, pMsg); + int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 7d7c01a870..ec75ffcae1 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -57,9 +57,7 @@ void sndMetaDelete(SStreamMeta *pMeta) { } int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { - for (int i = 0; i < pTask->exec.numOfRunners; i++) { - pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); - } + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index b10bee4732..4141485d28 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -48,7 +48,6 @@ target_sources( # tq "src/tq/tq.c" "src/tq/tqCommit.c" - "src/tq/tqMetaStore.c" "src/tq/tqOffset.c" "src/tq/tqPush.c" "src/tq/tqRead.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index db992f85d4..9e33973c05 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -51,7 +51,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodeClose(SVnode *pVnode); -int32_t vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version); +int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); @@ -126,7 +126,7 @@ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList); int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); -int32_t tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList); +int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); @@ -174,20 +174,20 @@ typedef struct { } STableKeyInfo; struct SMetaEntry { - int64_t version; - int8_t type; - tb_uid_t uid; - const char *name; + int64_t version; + int8_t type; + tb_uid_t uid; + char *name; union { struct { SSchemaWrapper schema; SSchemaWrapper schemaTag; } stbEntry; struct { - int64_t ctime; - int32_t ttlDays; - tb_uid_t suid; - const uint8_t *pTags; + int64_t ctime; + int32_t ttlDays; + tb_uid_t suid; + uint8_t *pTags; } ctbEntry; struct { int64_t ctime; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index a8a3e4f601..f89df4a96f 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -41,45 +41,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -#define TQ_BUFFER_SIZE 4 - -#define TQ_BUCKET_MASK 0xFF -#define TQ_BUCKET_SIZE 256 - -#define TQ_PAGE_SIZE 4096 -// key + offset + size -#define TQ_IDX_SIZE 24 -// 4096 / 24 -#define TQ_MAX_IDX_ONE_PAGE 170 -// 24 * 170 -#define TQ_IDX_PAGE_BODY_SIZE 4080 -// 4096 - 4080 -#define TQ_IDX_PAGE_HEAD_SIZE 16 - -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 -#define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 - -#define TQ_SVER 0 - -// TODO: inplace mode is not implemented -#define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 - -#define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 - -static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } - -static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } - -static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; - -#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE - -typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; - typedef struct STqOffsetCfg STqOffsetCfg; typedef struct STqOffsetStore STqOffsetStore; @@ -98,53 +59,6 @@ struct STqReadHandle { STSchema* pSchema; }; -typedef struct { - int16_t ver; - int16_t action; - int32_t checksum; - int64_t ssize; - char content[]; -} STqSerializedHead; - -typedef int32_t (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); -typedef int32_t (*FTqDeserialize)(void* self, const STqSerializedHead* pHead, void** ppObj); -typedef void (*FTqDelete)(void*); - -typedef struct { - int64_t key; - int64_t offset; - int64_t serializedSize; - void* valueInUse; - void* valueInTxn; -} STqMetaHandle; - -typedef struct STqMetaList { - STqMetaHandle handle; - struct STqMetaList* next; - // struct STqMetaList* inTxnPrev; - // struct STqMetaList* inTxnNext; - struct STqMetaList* unpersistPrev; - struct STqMetaList* unpersistNext; -} STqMetaList; - -typedef struct { - STQ* pTq; - STqMetaList* bucket[TQ_BUCKET_SIZE]; - // a table head - STqMetaList* unpersistHead; - // topics that are not connectted - STqMetaList* unconnectTopic; - - TdFilePtr pFile; - TdFilePtr pIdxFile; - - char* dirPath; - int32_t tqConfigFlag; - FTqSerialize pSerializer; - FTqDeserialize pDeserializer; - FTqDelete pDeleter; -} STqMetaStore; - typedef struct { int64_t consumerId; int32_t epoch; @@ -173,8 +87,7 @@ typedef struct { } STqExec; struct STQ { - char* path; - // STqMetaStore* tqMeta; + char* path; SHashObj* pushMgr; // consumerId -> STqExec* SHashObj* execs; // subKey -> STqExec SHashObj* pStreamTasks; @@ -190,87 +103,10 @@ typedef struct { static STqMgmt tqMgmt; -typedef struct { - int8_t status; - int64_t offset; - qTaskInfo_t task; - STqReadHandle* pReadHandle; -} STqTaskItem; - -// new version -typedef struct { - int64_t firstOffset; - int64_t lastOffset; - STqTaskItem output[TQ_BUFFER_SIZE]; -} STqBuffer; - -typedef struct { - char topicName[TSDB_TOPIC_FNAME_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - char* qmsg; - STqBuffer buffer; - SWalReadHandle* pReadhandle; -} STqTopic; - -typedef struct { - int64_t consumerId; - int32_t epoch; - char cgroup[TSDB_TOPIC_FNAME_LEN]; - SArray* topics; // SArray -} STqConsumer; - -typedef struct { - int8_t type; - int8_t nodeType; - int8_t reserved[6]; - int64_t streamId; - qTaskInfo_t task; - // TODO sync function -} STqStreamPusher; - -typedef struct { - int8_t inited; - tmr_h timer; -} STqPushMgmt; - -static STqPushMgmt tqPushMgmt; - // init once int tqInit(); void tqCleanUp(); -// open in each vnode -// required by vnode - -int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); -int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); - -static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } - -// tqMetaStore.h -STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, - FTqDelete pDeleter, int32_t tqConfigFlag); -int32_t tqStoreClose(STqMetaStore*); -// int32_t tqStoreDelete(TqMetaStore*); -// int32_t tqStoreCommitAll(TqMetaStore*); -int32_t tqStorePersist(STqMetaStore*); -// clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore*); - -void* tqHandleGet(STqMetaStore*, int64_t key); -// make it unpersist -void* tqHandleTouchGet(STqMetaStore*, int64_t key); -int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); -int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); -// delete committed kv pair -// notice that a delete action still needs to be committed -int32_t tqHandleDel(STqMetaStore*, int64_t key); -int32_t tqHandlePurge(STqMetaStore*, int64_t key); -int32_t tqHandleCommit(STqMetaStore*, int64_t key); -int32_t tqHandleAbort(STqMetaStore*, int64_t key); - // tqOffset STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); void STqOffsetClose(STqOffsetStore*); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 23825e6f4a..24b3f458b1 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -104,7 +104,7 @@ int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeep int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); int tsdbCommit(STsdb* pTsdb); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, const SSubmitReq* pMsg); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, @@ -123,11 +123,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); -#if 0 -int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); -int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); -#endif -int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data); +int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5a24f78ce8..a792343380 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -30,9 +30,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void * pBuf = NULL; + void *pBuf = NULL; int32_t szBuf = 0; - void * p = NULL; + void *p = NULL; SMetaReader mr = {0}; // validate req @@ -71,9 +71,9 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { - TBC * pNameIdxc = NULL; - TBC * pUidIdxc = NULL; - TBC * pCtbIdxc = NULL; + TBC *pNameIdxc = NULL; + TBC *pUidIdxc = NULL; + TBC *pCtbIdxc = NULL; SCtbIdxKey *pCtbIdxKey; const void *pKey = NULL; int nKey; @@ -134,8 +134,8 @@ _err: int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TBC * pUidIdxc = NULL; - TBC * pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -165,7 +165,9 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ASSERT(ret == 0); - tDecoderInit(&dc, pData, nData); + oStbEntry.pBuf = taosMemoryMalloc(nData); + memcpy(oStbEntry.pBuf, pData, nData); + tDecoderInit(&dc, oStbEntry.pBuf, nData); metaDecodeEntry(&dc, &oStbEntry); nStbEntry.version = version; @@ -193,6 +195,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // update uid index tdbTbcUpsert(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &version, sizeof(version), 0); + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); metaULock(pMeta); tDecoderClear(&dc); tdbTbcClose(pTbDbc); @@ -220,9 +223,6 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; metaReaderClear(&mr); return -1; - } else { - pReq->uid = tGenIdPI64(); - pReq->ctime = taosGetTimestampMs(); } metaReaderClear(&mr); @@ -256,9 +256,9 @@ _err: } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { - TBC * pTbDbc = NULL; - TBC * pUidIdxc = NULL; - TBC * pNameIdxc = NULL; + TBC *pTbDbc = NULL; + TBC *pUidIdxc = NULL; + TBC *pNameIdxc = NULL; const void *pData; int nData; tb_uid_t uid; @@ -377,14 +377,14 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { - void * pVal = NULL; + void *pVal = NULL; int nVal = 0; - const void * pData = NULL; + const void *pData = NULL; int nData = 0; int ret = 0; tb_uid_t uid; int64_t oversion; - SSchema * pColumn = NULL; + SSchema *pColumn = NULL; SMetaEntry entry = {0}; SSchemaWrapper *pSchema; int c; @@ -420,7 +420,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // get table entry SDecoder dc = {0}; - tDecoderInit(&dc, pData, nData); + entry.pBuf = taosMemoryMalloc(nData); + memcpy(entry.pBuf, pData, nData); + tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); ASSERT(ret == 0); @@ -530,7 +532,7 @@ _err: static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { SMetaEntry ctbEntry = {0}; SMetaEntry stbEntry = {0}; - void * pVal = NULL; + void *pVal = NULL; int nVal = 0; int ret; int c; @@ -561,7 +563,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA oversion = *(int64_t *)pData; // search table.db - TBC * pTbDbc = NULL; + TBC *pTbDbc = NULL; SDecoder dc1 = {0}; SDecoder dc2 = {0}; @@ -585,7 +587,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaDecodeEntry(&dc2, &stbEntry); SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema * pColumn = NULL; + SSchema *pColumn = NULL; int32_t iCol = 0; for (;;) { pColumn = NULL; @@ -690,8 +692,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void * pKey = NULL; - void * pVal = NULL; + void *pKey = NULL; + void *pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; @@ -806,14 +808,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void * pData = NULL; + void *pData = NULL; int nData = 0; STbDbKey tbDbKey = {0}; SMetaEntry stbEntry = {0}; - STagIdxKey * pTagIdxKey = NULL; + STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void * pTagData = NULL; // + const void *pTagData = NULL; // SDecoder dc = {0}; // get super table @@ -855,7 +857,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { SEncoder coder = {0}; - void * pVal = NULL; + void *pVal = NULL; int vLen = 0; int rcode = 0; SSkmDbKey skmDbKey = {0}; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 25fa716d4e..6ca523b580 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -36,15 +36,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { /*ASSERT(0);*/ /*}*/ -#if 0 - pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, - (FTqDelete)taosMemoryFree, 0); - if (pTq->tqMeta == NULL) { - taosMemoryFree(pTq); - return NULL; - } -#endif - pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -65,48 +56,6 @@ void tqClose(STQ* pTq) { // TODO } -static void tdSRowDemo() { -#define DEMO_N_COLS 3 - - int16_t schemaVersion = 0; - int32_t numOfCols = DEMO_N_COLS; // ts + int - SRowBuilder rb = {0}; - - SSchema schema[DEMO_N_COLS] = { - {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = COL_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = COL_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = COL_SMA_ON}}; - - SSchema* pSchema = schema; - STSchema* pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); - - tdSRowInit(&rb, schemaVersion); - tdSRowSetTpInfo(&rb, numOfCols, pTSChema->flen); - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSChema); - void* row = taosMemoryCalloc(1, maxLen); // make sure the buffer is enough - - // set row buf - tdSRowResetBuf(&rb, row); - - for (int32_t idx = 0; idx < pTSChema->numOfCols; ++idx) { - STColumn* pColumn = pTSChema->columns + idx; - if (idx == 0) { - int64_t tsKey = 1651234567; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &tsKey, true, pColumn->offset, idx); - } else if (idx == 1) { - int32_t val1 = 10; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &val1, true, pColumn->offset, idx); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, true, pColumn->offset, idx); - } - } - - // print - tdSRowPrint(row, pTSChema, __func__); - - taosMemoryFree(pTSChema); -} - int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { void* pIter = NULL; while (1) { @@ -261,166 +210,26 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ } int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - if (msgType != TDMT_VND_SUBMIT) return 0; + if (msgType == TDMT_VND_SUBMIT) { + if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; - // make sure msgType == TDMT_VND_SUBMIT - if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { - return -1; + if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) { + // TODO handle sma error + } + void* data = taosMemoryMalloc(msgLen); + if (data == NULL) { + return -1; + } + memcpy(data, msg, msgLen); + + tqProcessStreamTrigger(pTq, data); } - if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; - - void* data = taosMemoryMalloc(msgLen); - if (data == NULL) { - return -1; - } - memcpy(data, msg, msgLen); - - tqProcessStreamTriggerNew(pTq, data); - -#if 0 - SRpcMsg req = { - .msgType = TDMT_VND_STREAM_TRIGGER, - .pCont = data, - .contLen = msgLen, - }; - - tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req); -#endif - return 0; } int tqCommit(STQ* pTq) { // do nothing - /*return tqStorePersist(pTq->tqMeta);*/ - return 0; -} - -int32_t tqGetTopicHandleSize(const STqTopic* pTopic) { - return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) + - sizeof(int64_t) * 3; -} - -int32_t tqGetConsumerHandleSize(const STqConsumer* pConsumer) { - int num = taosArrayGetSize(pConsumer->topics); - int32_t sz = 0; - for (int i = 0; i < num; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - sz += tqGetTopicHandleSize(pTopic); - } - return sz; -} - -static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopic->topicName); - /*tlen += taosEncodeString(buf, pTopic->sql);*/ - /*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/ - tlen += taosEncodeString(buf, pTopic->qmsg); - /*tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset);*/ - /*tlen += taosEncodeFixedI64(buf, pTopic->committedOffset);*/ - /*tlen += taosEncodeFixedI64(buf, pTopic->currentOffset);*/ - return tlen; -} - -static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopic) { - buf = taosDecodeStringTo(buf, pTopic->topicName); - /*buf = taosDecodeString(buf, &pTopic->sql);*/ - /*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/ - buf = taosDecodeString(buf, &pTopic->qmsg); - /*buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset);*/ - /*buf = taosDecodeFixedI64(buf, &pTopic->committedOffset);*/ - /*buf = taosDecodeFixedI64(buf, &pTopic->currentOffset);*/ - return buf; -} - -static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pConsumer) { - int32_t sz; - - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); - tlen += taosEncodeFixedI32(buf, pConsumer->epoch); - tlen += taosEncodeString(buf, pConsumer->cgroup); - sz = taosArrayGetSize(pConsumer->topics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - tlen += tEncodeSTqTopic(buf, pTopic); - } - return tlen; -} - -static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer* pConsumer) { - int32_t sz; - - buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); - buf = taosDecodeFixedI32(buf, &pConsumer->epoch); - buf = taosDecodeStringTo(buf, pConsumer->cgroup); - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic)); - if (pConsumer->topics == NULL) return NULL; - for (int32_t i = 0; i < sz; i++) { - STqTopic pTopic; - buf = tDecodeSTqTopic(buf, &pTopic); - taosArrayPush(pConsumer->topics, &pTopic); - } - return buf; -} - -int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead) { - int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); - - if (sz > (*ppHead)->ssize) { - void* tmpPtr = taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sz); - if (tmpPtr == NULL) { - taosMemoryFree(*ppHead); - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - *ppHead = tmpPtr; - (*ppHead)->ssize = sz; - } - - void* ptr = (*ppHead)->content; - void* abuf = ptr; - tEncodeSTqConsumer(&abuf, pConsumer); - - return 0; -} - -int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { - const void* str = pHead->content; - *ppConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); - if (*ppConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - if (tDecodeSTqConsumer(str, *ppConsumer) == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - STqConsumer* pConsumer = *ppConsumer; - int32_t sz = taosArrayGetSize(pConsumer->topics); - for (int32_t i = 0; i < sz; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); - if (pTopic->pReadhandle == NULL) { - ASSERT(false); - } - for (int j = 0; j < TQ_BUFFER_SIZE; j++) { - pTopic->buffer.output[j].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - }; - pTopic->buffer.output[j].pReadHandle = pReadHandle; - pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); - } - } - return 0; } @@ -685,213 +494,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; } -#if 0 -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { - SMqPollReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t fetchOffset; - int64_t blockingTime = pReq->blockingTime; - int32_t reqEpoch = pReq->epoch; - - if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = walGetFirstVer(pTq->pWal); - } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { - fetchOffset = walGetLastVer(pTq->pWal); - } else { - fetchOffset = pReq->currentOffset + 1; - } - - tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); - - SMqPollRspV2 rspV2 = {0}; - rspV2.dataLen = 0; - - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - if (pConsumer == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode)); - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - tmsgSendRsp(pMsg); - return 0; - } - - int32_t consumerEpoch = atomic_load_32(&pConsumer->epoch); - while (consumerEpoch < reqEpoch) { - consumerEpoch = atomic_val_compare_exchange_32(&pConsumer->epoch, consumerEpoch, reqEpoch); - } - - STqTopic* pTopic = NULL; - int32_t topicSz = taosArrayGetSize(pConsumer->topics); - for (int32_t i = 0; i < topicSz; i++) { - STqTopic* topic = taosArrayGet(pConsumer->topics, i); - // TODO race condition - ASSERT(pConsumer->consumerId == consumerId); - if (strcmp(topic->topicName, pReq->topic) == 0) { - pTopic = topic; - break; - } - } - if (pTopic == NULL) { - vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic, - TD_VID(pTq->pVnode)); - pMsg->pCont = NULL; - pMsg->contLen = 0; - pMsg->code = -1; - tmsgSendRsp(pMsg); - return 0; - } - - tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch, - TD_VID(pTq->pVnode)); - - rspV2.reqOffset = pReq->currentOffset; - rspV2.skipLogNum = 0; - - while (1) { - /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/ - // TODO - consumerEpoch = atomic_load_32(&pConsumer->epoch); - if (consumerEpoch > reqEpoch) { - tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); - break; - } - SWalReadHead* pHead; - if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) { - // TODO: no more log, set timer to wait blocking time - // if data inserted during waiting, launch query and - // response to user - tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset); - break; - } - tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); - /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/ - /*pHead = pTopic->pReadhandle->pHead;*/ - if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - qTaskInfo_t task = pTopic->buffer.output[workerId].task; - ASSERT(task); - qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK); - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - while (1) { - SSDataBlock* pDataBlock = NULL; - uint64_t ts; - if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(false); - } - if (pDataBlock == NULL) { - /*pos = fetchOffset % TQ_BUFFER_SIZE;*/ - break; - } - - taosArrayPush(pRes, pDataBlock); - } - - if (taosArrayGetSize(pRes) == 0) { - tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId, - pReq->epoch, TD_VID(pTq->pVnode), fetchOffset); - fetchOffset++; - rspV2.skipLogNum++; - taosArrayDestroy(pRes); - continue; - } - rspV2.rspOffset = fetchOffset; - - int32_t blockSz = taosArrayGetSize(pRes); - int32_t dataBlockStrLen = 0; - for (int32_t i = 0; i < blockSz; i++) { - SSDataBlock* pBlock = taosArrayGet(pRes, i); - dataBlockStrLen += sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); - } - - void* dataBlockBuf = taosMemoryMalloc(dataBlockStrLen); - if (dataBlockBuf == NULL) { - pMsg->code = -1; - taosMemoryFree(pHead); - } - - rspV2.blockData = dataBlockBuf; - - int32_t pos; - rspV2.blockPos = taosArrayInit(blockSz, sizeof(int32_t)); - for (int32_t i = 0; i < blockSz; i++) { - pos = 0; - SSDataBlock* pBlock = taosArrayGet(pRes, i); - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)dataBlockBuf; - pRetrieve->useconds = 0; - pRetrieve->precision = 0; - pRetrieve->compressed = 0; - pRetrieve->completed = 1; - pRetrieve->numOfRows = htonl(pBlock->info.rows); - blockCompressEncode(pBlock, pRetrieve->data, &pos, pBlock->info.numOfCols, false); - taosArrayPush(rspV2.blockPos, &rspV2.dataLen); - - int32_t totLen = sizeof(SRetrieveTableRsp) + pos; - pRetrieve->compLen = htonl(totLen); - rspV2.dataLen += totLen; - dataBlockBuf = POINTER_SHIFT(dataBlockBuf, totLen); - } - ASSERT(POINTER_DISTANCE(dataBlockBuf, rspV2.blockData) <= dataBlockStrLen); - - int32_t msgLen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2); - void* buf = rpcMallocCont(msgLen); - - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* msgBodyBuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRspV2(&msgBodyBuf, &rspV2); - - /*rsp.pBlockData = pRes;*/ - - /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ - SRpcMsg resp = {.info = pMsg->info, pCont = buf, .contLen = msgLen, .code = 0}; - tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset, - pHead->msgType, consumerId, pReq->epoch); - tmsgSendRsp(&resp); - taosMemoryFree(pHead); - return 0; - } else { - taosMemoryFree(pHead); - fetchOffset++; - rspV2.skipLogNum++; - } - } - - /*if (blockingTime != 0) {*/ - /*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/ - /*} else {*/ - - rspV2.rspOffset = fetchOffset - 1; - - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRspV2(NULL, &rspV2); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - return -1; - } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqPollRspV2(&abuf, &rspV2); - - SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; - tmsgSendRsp(&resp); - tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId, - pReq->epoch); - /*}*/ - - return 0; -} -#endif - int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; @@ -981,55 +583,6 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0); } -int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { - pTask->status = TASK_STATUS__IDLE; - pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; - pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; - - pTask->inputQ = taosOpenQueue(); - pTask->outputQ = taosOpenQueue(); - pTask->inputQAll = taosAllocateQall(); - pTask->outputQAll = taosAllocateQall(); - - if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) - goto FAIL; - - if (pTask->execType != TASK_EXEC__NONE) { - // expand runners - pTask->exec.numOfRunners = parallel; - pTask->exec.runners = taosMemoryCalloc(parallel, sizeof(SStreamRunner)); - if (pTask->exec.runners == NULL) { - goto FAIL; - } - for (int32_t i = 0; i < parallel; i++) { - STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - SReadHandle handle = { - .reader = pStreamReader, - .meta = pTq->pVnode->pMeta, - .pMsgCb = &pTq->pVnode->msgCb, - .vnode = pTq->pVnode, - }; - pTask->exec.runners[i].inputHandle = pStreamReader; - pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); - ASSERT(pTask->exec.runners[i].executor); - } - } - - if (pTask->sinkType == TASK_SINK__TABLE) { - pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink; - } - - return 0; -FAIL: - if (pTask->inputQ) taosCloseQueue(pTask->inputQ); - if (pTask->outputQ) taosCloseQueue(pTask->outputQ); - if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); - if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); - if (pTask) taosMemoryFree(pTask); - return -1; -} - int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { @@ -1042,9 +595,31 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { } tDecoderClear(&decoder); + pTask->status = TASK_STATUS__IDLE; + pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; + pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + + pTask->inputQ = taosOpenQueue(); + pTask->outputQ = taosOpenQueue(); + pTask->inputQAll = taosAllocateQall(); + pTask->outputQAll = taosAllocateQall(); + + if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) + goto FAIL; + // exec - if (tqExpandTask(pTq, pTask, 4) < 0) { - ASSERT(0); + if (pTask->execType != TASK_EXEC__NONE) { + // expand runners + STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + SReadHandle handle = { + .reader = pStreamReader, + .meta = pTq->pVnode->pMeta, + .pMsgCb = &pTq->pVnode->msgCb, + .vnode = pTq->pVnode, + }; + pTask->exec.inputHandle = pStreamReader; + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); + ASSERT(pTask->exec.executor); } // sink @@ -1052,8 +627,12 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { if (pTask->sinkType == TASK_SINK__SMA) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->sinkType == TASK_SINK__TABLE) { + pTask->tbSink.vnode = pTq->pVnode; + pTask->tbSink.tbSinkFunc = tqTableSink; + ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); + pTask->tbSink.pTSchema = tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols); ASSERT(pTask->tbSink.pTSchema); @@ -1061,94 +640,17 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); - return 0; -} - -int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId) { - void* pIter = NULL; - - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; - - if (streamExecTask(pTask, &pTq->pVnode->msgCb, data, STREAM_DATA_TYPE_SUBMIT_BLOCK, workerId) < 0) { - // TODO - } - } - return 0; -} - -#if 0 -int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* data) { - SStreamDataSubmit* pSubmit = NULL; - - // build data - pSubmit = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); - if (pSubmit == NULL) return -1; - pSubmit->dataRef = taosMemoryMalloc(sizeof(int32_t)); - if (pSubmit->dataRef == NULL) goto FAIL; - *pSubmit->dataRef = 1; - pSubmit->data = data; - pSubmit->type = STREAM_INPUT__DATA_BLOCK; - - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; - if (pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK) { - streamEnqueueDataSubmit(pTask, pSubmit); - // TODO cal back pressure - } - // check run - int8_t execStatus = atomic_load_8(&pTask->status); - if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { - SStreamTaskRunReq* pReq = taosMemoryMalloc(sizeof(SStreamTaskRunReq)); - if (pReq == NULL) continue; - // TODO: do we need htonl? - pReq->head.vgId = pTq->pVnode->config.vgId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; - SRpcMsg msg = { - .msgType = 0, - .pCont = pReq, - .contLen = sizeof(SStreamTaskRunReq), - }; - tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &msg); - } - } - streamDataSubmitRefDec(pSubmit); - return 0; FAIL: - if (pSubmit) { - if (pSubmit->dataRef) { - taosMemoryFree(pSubmit->dataRef); - } - taosFreeQitem(pSubmit); - } + if (pTask->inputQ) taosCloseQueue(pTask->inputQ); + if (pTask->outputQ) taosCloseQueue(pTask->outputQ); + if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); + if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); + if (pTask) taosMemoryFree(pTask); return -1; } -#endif -int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId) { - SStreamTaskExecReq req; - tDecodeSStreamTaskExecReq(msg, &req); - - int32_t taskId = req.taskId; - ASSERT(taskId); - - SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - ASSERT(pTask); - - if (streamExecTask(pTask, &pTq->pVnode->msgCb, req.data, STREAM_DATA_TYPE_SSDATA_BLOCK, workerId) < 0) { - // TODO - } - return 0; -} - -int32_t tqProcessStreamTriggerNew(STQ* pTq, SSubmitReq* pReq) { +int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { void* pIter = NULL; bool failed = false; @@ -1234,7 +736,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - streamTaskProcessDispatchReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); + streamProcessDispatchReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); return 0; } @@ -1242,7 +744,7 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - streamTaskProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); + streamProcessRecoverReq(pTask, &pTq->pVnode->msgCb, pReq, pMsg); return 0; } @@ -1250,7 +752,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->taskId; SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - streamTaskProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp); + streamProcessDispatchRsp(pTask, &pTq->pVnode->msgCb, pRsp); return 0; } @@ -1258,6 +760,6 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->taskId; SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - streamTaskProcessRecoverRsp(pTask, pRsp); + streamProcessRecoverRsp(pTask, pRsp); return 0; } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c deleted file mode 100644 index ca09cc1dc1..0000000000 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ /dev/null @@ -1,622 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tq.h" -// #include -// #include -// #include -// #include "osDir.h" - -#define TQ_META_NAME "tq.meta" -#define TQ_IDX_NAME "tq.idx" - -static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value); -static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key); - -static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { - if (pNode->unpersistNext == NULL) { - pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; - pNode->unpersistPrev = pMeta->unpersistHead; - pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode; - pMeta->unpersistHead->unpersistNext = pNode; - } -} - -static inline int64_t tqSeekLastPage(TdFilePtr pFile) { - int offset = taosLSeekFile(pFile, 0, SEEK_END); - int pageNo = offset / TQ_PAGE_SIZE; - int curPageOffset = pageNo * TQ_PAGE_SIZE; - return taosLSeekFile(pFile, curPageOffset, SEEK_SET); -} - -// TODO: the struct is tightly coupled with index entry -typedef struct STqIdxPageHead { - int16_t writeOffset; - int8_t unused[14]; -} STqIdxPageHead; - -typedef struct STqIdxPageBuf { - STqIdxPageHead head; - char buffer[TQ_IDX_PAGE_BODY_SIZE]; -} STqIdxPageBuf; - -static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(pFile); - int nBytes; - if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (nBytes == 0) { - memset(pBuf, 0, TQ_PAGE_SIZE); - pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; - } - ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - - return taosLSeekFile(pFile, offset, SEEK_SET); -} - -STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, - FTqDelete deleter, int32_t tqConfigFlag) { - STqMetaStore* pMeta = taosMemoryCalloc(1, sizeof(STqMetaStore)); - if (pMeta == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return NULL; - } - pMeta->pTq = pTq; - - // concat data file name and index file name - size_t pathLen = strlen(path); - pMeta->dirPath = taosMemoryMalloc(pathLen + 1); - if (pMeta->dirPath == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - taosMemoryFree(pMeta); - return NULL; - } - strcpy(pMeta->dirPath, path); - - char* name = taosMemoryMalloc(pathLen + 10); - - strcpy(name, path); - if (!taosDirExist(name) && taosMkDir(name) != 0) { - terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; - tqError("failed to create dir:%s since %s ", name, terrstr()); - } - strcat(name, "/" TQ_IDX_NAME); - TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ); - if (pIdxFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - tqError("failed to open file:%s since %s ", name, terrstr()); - // free memory - taosMemoryFree(name); - return NULL; - } - - pMeta->pIdxFile = pIdxFile; - pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pMeta->unpersistHead == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - taosMemoryFree(name); - return NULL; - } - pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; - - strcpy(name, path); - strcat(name, "/" TQ_META_NAME); - TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ); - if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - tqError("failed to open file:%s since %s", name, terrstr()); - taosMemoryFree(name); - return NULL; - } - taosMemoryFree(name); - - pMeta->pFile = pFile; - - pMeta->pSerializer = serializer; - pMeta->pDeserializer = deserializer; - pMeta->pDeleter = deleter; - pMeta->tqConfigFlag = tqConfigFlag; - - // read idx file and load into memory - STqIdxPageBuf idxBuf; - STqSerializedHead* serializedObj = taosMemoryMalloc(TQ_PAGE_SIZE); - if (serializedObj == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - } - int idxRead; - int allocated = TQ_PAGE_SIZE; - bool readEnd = false; - while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { - if (idxRead == -1) { - // TODO: handle error - terrno = TAOS_SYSTEM_ERROR(errno); - tqError("failed to read tq index file since %s", terrstr()); - } - ASSERT(idxBuf.head.writeOffset == idxRead); - // loop read every entry - for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pNode == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // TODO: free memory - } - memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - - taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); - if (allocated < pNode->handle.serializedSize) { - void* ptr = taosMemoryRealloc(serializedObj, pNode->handle.serializedSize); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // TODO: free memory - } - serializedObj = ptr; - allocated = pNode->handle.serializedSize; - } - serializedObj->ssize = pNode->handle.serializedSize; - if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { - // TODO: read error - } - if (serializedObj->action == TQ_ACTION_INUSE) { - if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); - } else { - pNode->handle.valueInUse = TQ_DELETE_TOKEN; - } - } else if (serializedObj->action == TQ_ACTION_INTXN) { - if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInTxn); - } else { - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; - } - } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) { - if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); - } else { - pNode->handle.valueInUse = TQ_DELETE_TOKEN; - } - STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); - if (ptr->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(pTq, ptr, &pNode->handle.valueInTxn); - } else { - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; - } - } else { - ASSERT(0); - } - - // put into list - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; - STqMetaList* pBucketNode = pMeta->bucket[bucketKey]; - if (pBucketNode == NULL) { - pMeta->bucket[bucketKey] = pNode; - } else if (pBucketNode->handle.key == pNode->handle.key) { - pNode->next = pBucketNode->next; - pMeta->bucket[bucketKey] = pNode; - } else { - while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) { - pBucketNode = pBucketNode->next; - } - if (pBucketNode->next) { - ASSERT(pBucketNode->next->handle.key == pNode->handle.key); - STqMetaList* pNodeFound = pBucketNode->next; - pNode->next = pNodeFound->next; - pBucketNode->next = pNode; - pBucketNode = pNodeFound; - } else { - pNode->next = pMeta->bucket[bucketKey]; - pMeta->bucket[bucketKey] = pNode; - pBucketNode = NULL; - } - } - if (pBucketNode) { - if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pBucketNode->handle.valueInUse); - } - if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pBucketNode->handle.valueInTxn); - } - taosMemoryFree(pBucketNode); - } - } - } - taosMemoryFree(serializedObj); - return pMeta; -} - -int32_t tqStoreClose(STqMetaStore* pMeta) { - // commit data and idx - tqStorePersist(pMeta); - ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - taosCloseFile(&pMeta->pFile); - taosCloseFile(&pMeta->pIdxFile); - // free memory - for (int i = 0; i < TQ_BUCKET_SIZE; i++) { - STqMetaList* pNode = pMeta->bucket[i]; - while (pNode) { - ASSERT(pNode->unpersistNext == NULL); - ASSERT(pNode->unpersistPrev == NULL); - if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - STqMetaList* next = pNode->next; - taosMemoryFree(pNode); - pNode = next; - } - } - taosMemoryFree(pMeta->dirPath); - taosMemoryFree(pMeta->unpersistHead); - taosMemoryFree(pMeta); - return 0; -} - -int32_t tqStoreDelete(STqMetaStore* pMeta) { - taosCloseFile(&pMeta->pFile); - taosCloseFile(&pMeta->pIdxFile); - // free memory - for (int i = 0; i < TQ_BUCKET_SIZE; i++) { - STqMetaList* pNode = pMeta->bucket[i]; - pMeta->bucket[i] = NULL; - while (pNode) { - if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - STqMetaList* next = pNode->next; - taosMemoryFree(pNode); - pNode = next; - } - } - taosMemoryFree(pMeta->unpersistHead); - taosRemoveDir(pMeta->dirPath); - taosMemoryFree(pMeta->dirPath); - taosMemoryFree(pMeta); - return 0; -} - -int32_t tqStorePersist(STqMetaStore* pMeta) { - STqIdxPageBuf idxBuf; - int64_t* bufPtr = (int64_t*)idxBuf.buffer; - STqMetaList* pHead = pMeta->unpersistHead; - STqMetaList* pNode = pHead->unpersistNext; - STqSerializedHead* pSHead = taosMemoryMalloc(sizeof(STqSerializedHead)); - if (pSHead == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - pSHead->ver = TQ_SVER; - pSHead->checksum = 0; - pSHead->ssize = sizeof(STqSerializedHead); - /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - - tqReadLastPage(pMeta->pIdxFile, &idxBuf); - - if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); - memset(&idxBuf, 0, TQ_PAGE_SIZE); - idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; - } else { - bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset); - } - - while (pHead != pNode) { - int nBytes = 0; - - if (pNode->handle.valueInUse) { - if (pNode->handle.valueInTxn) { - pSHead->action = TQ_ACTION_INUSE_CONT; - } else { - pSHead->action = TQ_ACTION_INUSE; - } - - if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(STqSerializedHead); - } else { - pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); - } - nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); - ASSERT(nBytes == pSHead->ssize); - } - - if (pNode->handle.valueInTxn) { - pSHead->action = TQ_ACTION_INTXN; - if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(STqSerializedHead); - } else { - pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); - } - int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); - ASSERT(nBytesTxn == pSHead->ssize); - nBytes += nBytesTxn; - } - pNode->handle.offset = offset; - offset += nBytes; - - // write idx file - // TODO: endian check and convert - *(bufPtr++) = pNode->handle.key; - *(bufPtr++) = pNode->handle.offset; - *(bufPtr++) = (int64_t)nBytes; - idxBuf.head.writeOffset += TQ_IDX_SIZE; - - if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); - // TODO: handle error with tfile - ASSERT(nBytes == TQ_PAGE_SIZE); - memset(&idxBuf, 0, TQ_PAGE_SIZE); - idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; - bufPtr = (int64_t*)&idxBuf.buffer; - } - // remove from unpersist list - pHead->unpersistNext = pNode->unpersistNext; - pHead->unpersistNext->unpersistPrev = pHead; - pNode->unpersistPrev = pNode->unpersistNext = NULL; - pNode = pHead->unpersistNext; - - // remove from bucket - if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) { - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; - STqMetaList* pBucketHead = pMeta->bucket[bucketKey]; - if (pBucketHead == pNode) { - pMeta->bucket[bucketKey] = pNode->next; - } else { - STqMetaList* pBucketNode = pBucketHead; - while (pBucketNode->next != NULL && pBucketNode->next != pNode) { - pBucketNode = pBucketNode->next; - } - // impossible for pBucket->next == NULL - ASSERT(pBucketNode->next == pNode); - pBucketNode->next = pNode->next; - } - taosMemoryFree(pNode); - } - } - - // write left bytes - taosMemoryFree(pSHead); - // TODO: write new version in tfile - if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); - // TODO: handle error in tfile - ASSERT(nBytes == idxBuf.head.writeOffset); - } - // TODO: using fsync in tfile - taosFsyncFile(pMeta->pIdxFile); - taosFsyncFile(pMeta->pFile); - return 0; -} - -static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - // change pointer ownership - pNode->handle.valueInUse = value; - return 0; - } else { - pNode = pNode->next; - } - } - STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pNewNode == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - pNewNode->handle.key = key; - pNewNode->handle.valueInUse = value; - pNewNode->next = pMeta->bucket[bucketKey]; - // put into unpersist list - pNewNode->unpersistPrev = pMeta->unpersistHead; - pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; - pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode; - pMeta->unpersistHead->unpersistNext = pNewNode; - return 0; -} - -void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - return pNode->handle.valueInUse; - } else { - return NULL; - } - } else { - pNode = pNode->next; - } - } - return NULL; -} - -void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - tqLinkUnpersist(pMeta, pNode); - return pNode->handle.valueInUse; - } else { - return NULL; - } - } else { - pNode = pNode->next; - } - } - return NULL; -} - -static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn) { - if (tqDupIntxnReject(pMeta->tqConfigFlag)) { - terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN; - return -1; - } - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - } - pNode->handle.valueInTxn = value; - tqLinkUnpersist(pMeta, pNode); - return 0; - } else { - pNode = pNode->next; - } - } - STqMetaList* pNewNode = taosMemoryCalloc(1, sizeof(STqMetaList)); - if (pNewNode == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - pNewNode->handle.key = key; - pNewNode->handle.valueInTxn = value; - pNewNode->next = pMeta->bucket[bucketKey]; - pMeta->bucket[bucketKey] = pNewNode; - tqLinkUnpersist(pMeta, pNewNode); - return 0; -} - -int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } - -int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void* vmem = taosMemoryMalloc(vsize); - if (vmem == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - memcpy(vmem, value, vsize); - return tqHandlePutImpl(pMeta, key, vmem); -} - -static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - return pNode->handle.valueInTxn; - } else { - return NULL; - } - } else { - pNode = pNode->next; - } - } - return NULL; -} - -int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn == NULL) { - terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; - return -1; - } - if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInUse); - } - pNode->handle.valueInUse = pNode->handle.valueInTxn; - pNode->handle.valueInTxn = NULL; - tqLinkUnpersist(pMeta, pNode); - return 0; - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - pNode->handle.valueInTxn = NULL; - tqLinkUnpersist(pMeta, pNode); - return 0; - } - terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; - return -1; - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if (pNode->handle.valueInTxn) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } - - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; - tqLinkUnpersist(pMeta, pNode); - return 0; - } - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; - STqMetaList* pNode = pMeta->bucket[bucketKey]; - while (pNode) { - if (pNode->handle.key == key) { - pNode->handle.valueInUse = TQ_DELETE_TOKEN; - tqLinkUnpersist(pMeta, pNode); - return 0; - } else { - pNode = pNode->next; - } - } - terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; - return -1; -} - -// TODO: clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 2acca738fb..025b2ab580 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -38,7 +38,7 @@ struct SMemTable { struct SMemSkipListNode { int8_t level; - SMemSkipListNode *forwards[1]; // Windows does not allow 0 + SMemSkipListNode *forwards[1]; // Windows does not allow 0 }; struct SMemSkipList { @@ -46,7 +46,7 @@ struct SMemSkipList { int8_t maxLevel; int8_t level; int32_t size; - SMemSkipListNode pHead[1]; // Windows does not allow 0 + SMemSkipListNode pHead[1]; // Windows does not allow 0 }; struct SMemData { @@ -217,7 +217,7 @@ int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *p if (tDecodeIsEnd(&dc)) break; // decode row - if (tDecodeBinary(&dc, (const uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { + if (tDecodeBinary(&dc, (uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -273,7 +273,7 @@ static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pR static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) { if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1; - if (tDecodeBinary(pDecoder, (const uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; + if (tDecodeBinary(pDecoder, (uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index a67f413ba7..aab4da26a3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -85,7 +85,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro return 0; } -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -150,7 +150,6 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, const SSubmitReq *pMsg) { return -1; } } - } if (terrno != TSDB_CODE_SUCCESS) return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 88cc97ddd5..5e50a1b796 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -24,26 +24,66 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { -#if 0 - SRpcMsg *pMsg; - SRpcMsg *pRpc; +int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { + SDecoder dc = {0}; - *version = pVnode->state.processed; - for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - pRpc = pMsg; + switch (pMsg->msgType) { + case TDMT_VND_CREATE_TABLE: { + int64_t ctime = taosGetTimestampMs(); + int32_t nReqs; - // set request version - if (walWrite(pVnode->pWal, pVnode->state.processed++, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { - vError("vnode:%d write wal error since %s", TD_VID(pVnode), terrstr()); - return -1; - } + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + tStartDecode(&dc); + + tDecodeI32v(&dc, &nReqs); + for (int32_t iReq = 0; iReq < nReqs; iReq++) { + tb_uid_t uid = tGenIdPI64(); + tStartDecode(&dc); + + tDecodeI32v(&dc, NULL); + *(int64_t *)(dc.data + dc.pos) = uid; + *(int64_t *)(dc.data + dc.pos + 8) = ctime; + + tEndDecode(&dc); + } + + tEndDecode(&dc); + tDecoderClear(&dc); + } break; + case TDMT_VND_SUBMIT: { + SSubmitMsgIter msgIter = {0}; + SSubmitReq *pSubmitReq = (SSubmitReq *)pMsg->pCont; + SSubmitBlk *pBlock = NULL; + int64_t ctime = taosGetTimestampMs(); + tb_uid_t uid; + + tInitSubmitMsgIter(pSubmitReq, &msgIter); + + for (;;) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + + if (msgIter.schemaLen > 0) { + uid = tGenIdPI64(); + + tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); + tStartDecode(&dc); + + tDecodeI32v(&dc, NULL); + *(int64_t *)(dc.data + dc.pos) = uid; + *(int64_t *)(dc.data + dc.pos + 8) = ctime; + pBlock->uid = htobe64(uid); + + tEndDecode(&dc); + tDecoderClear(&dc); + } + } + + } break; + default: + break; } - walFsync(pVnode->pWal, false); - -#endif return 0; } @@ -106,13 +146,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pMsg->contLen - sizeof(SMsgHead)) < 0) { } } break; -#if 0 - case TDMT_VND_TASK_WRITE_EXEC: { - if (tqProcessTaskExec(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead), - 0) < 0) { - } - } break; -#endif case TDMT_VND_ALTER_VNODE: break; default: @@ -195,17 +228,6 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_VND_TASK_RECOVER_RSP: return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg); -#if 0 - case TDMT_VND_TASK_PIPE_EXEC: - case TDMT_VND_TASK_MERGE_EXEC: - return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0); - case TDMT_VND_STREAM_TRIGGER:{ - // refactor, avoid double free - int code = tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0); - pMsg->pCont = NULL; - return code; - } -#endif case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: @@ -675,7 +697,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in goto _exit; } - for (int i = 0;;) { + for (;;) { tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; diff --git a/source/dnode/vnode/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp deleted file mode 100644 index 168daeb19f..0000000000 --- a/source/dnode/vnode/test/tqMetaTest.cpp +++ /dev/null @@ -1,279 +0,0 @@ -#include -#include -#include -#include - -#include "tqMetaStore.h" - -struct Foo { - int32_t a; -}; - -int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { - Foo* foo = (Foo*)pObj; - if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { - *ppHead = (STqSerializedHead*)taosMemoryRealloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); - (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); - } - *(int32_t*)(*ppHead)->content = foo->a; - return (*ppHead)->ssize; -} - -const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { - if (*ppObj == NULL) { - *ppObj = taosMemoryRealloc(*ppObj, sizeof(int32_t)); - } - Foo* pFoo = *(Foo**)ppObj; - pFoo->a = *(int32_t*)pHead->content; - return NULL; -} - -void FooDeleter(void* pObj) { taosMemoryFree(pObj); } - -class TqMetaUpdateAppendTest : public ::testing::Test { - protected: - void SetUp() override { - taosRemoveDir(pathName); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - } - - void TearDown() override { tqStoreClose(pMeta); } - - STqMetaStore* pMeta; - const char* pathName = TD_TMP_DIR_PATH "tq_test"; -}; - -TEST_F(TqMetaUpdateAppendTest, copyPutTest) { - Foo foo; - foo.a = 3; - tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo)); - - Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqHandleCommit(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo->a, 3); -} - -TEST_F(TqMetaUpdateAppendTest, persistTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 2; - tqHandleMovePut(pMeta, 1, pFoo); - Foo* pBar = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pBar == NULL, true); - tqHandleCommit(pMeta, 1); - pBar = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pBar->a, pFoo->a); - pBar = (Foo*)tqHandleGet(pMeta, 2); - EXPECT_EQ(pBar == NULL, true); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pBar = (Foo*)tqHandleGet(pMeta, 1); - ASSERT_EQ(pBar != NULL, true); - EXPECT_EQ(pBar->a, 2); - - pBar = (Foo*)tqHandleGet(pMeta, 2); - EXPECT_EQ(pBar == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, abortTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqHandleAbort(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, deleteTest) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqHandleCommit(pMeta, 1); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - ASSERT_EQ(pFoo != NULL, true); - EXPECT_EQ(pFoo->a, 3); - - tqHandleDel(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - ASSERT_EQ(pFoo != NULL, true); - EXPECT_EQ(pFoo->a, 3); - - tqHandleCommit(pMeta, 1); - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pFoo = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo == NULL, true); -} - -TEST_F(TqMetaUpdateAppendTest, intxnPersist) { - Foo* pFoo = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pFoo->a = 3; - tqHandleMovePut(pMeta, 1, pFoo); - tqHandleCommit(pMeta, 1); - - Foo* pBar = (Foo*)taosMemoryMalloc(sizeof(Foo)); - pBar->a = 4; - tqHandleMovePut(pMeta, 1, pBar); - - Foo* pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 3); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 3); - - tqHandleCommit(pMeta, 1); - - pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 4); - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - pFoo1 = (Foo*)tqHandleGet(pMeta, 1); - EXPECT_EQ(pFoo1->a, 4); -} - -TEST_F(TqMetaUpdateAppendTest, multiplePage) { - taosSeedRand(0); - std::vector v; - for (int i = 0; i < 1000; i++) { - v.push_back(taosRand()); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - for (int i = 0; i < 500; i++) { - tqHandleCommit(pMeta, i); - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - for (int i = 500; i < 1000; i++) { - tqHandleCommit(pMeta, i); - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } - - for (int i = 0; i < 1000; i++) { - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } -} - -TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { - taosSeedRand(0); - std::vector v; - for (int i = 0; i < 1000; i++) { - v.push_back(taosRand()); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 0; i < 500; i++) { - tqHandleCommit(pMeta, i); - v[i] = taosRand(); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 500; i < 1000; i++) { - v[i] = taosRand(); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 0; i < 1000; i++) { - tqHandleCommit(pMeta, i); - } - - tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); - ASSERT(pMeta); - - for (int i = 500; i < 1000; i++) { - v[i] = taosRand(); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - tqHandleCommit(pMeta, i); - } - - for (int i = 0; i < 1000; i++) { - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } -} - -TEST_F(TqMetaUpdateAppendTest, dupCommit) { - taosSeedRand(0); - std::vector v; - for (int i = 0; i < 1000; i++) { - v.push_back(taosRand()); - Foo foo; - foo.a = v[i]; - tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); - } - - for (int i = 0; i < 1000; i++) { - int ret = tqHandleCommit(pMeta, i); - EXPECT_EQ(ret, 0); - ret = tqHandleCommit(pMeta, i); - EXPECT_EQ(ret, -1); - } - - for (int i = 0; i < 1000; i++) { - int ret = tqHandleCommit(pMeta, i); - EXPECT_EQ(ret, -1); - } - - for (int i = 0; i < 1000; i++) { - Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); - ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; - EXPECT_EQ(pFoo->a, v[i]); - } -} diff --git a/source/libs/executor/inc/indexoperator.h b/source/libs/executor/inc/indexoperator.h deleted file mode 100644 index d033c63ef8..0000000000 --- a/source/libs/executor/inc/indexoperator.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _INDEX_OPERATOR_H -#define _INDEX_OPERATOR_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "nodes.h" -#include "tglobal.h" - -typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; - -SIdxFltStatus idxGetFltStatus(SNode *pFilterNode); -// construct tag filter operator later -int32_t doFilterTag(const SNode *pFilterNode, SArray *result); - -#ifdef __cplusplus -} -#endif - -#endif /*INDEX_OPERATOR_*/ diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 3e71888bf9..faab012f04 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -14,8 +14,8 @@ */ #include "builtins.h" -#include "querynodes.h" #include "builtinsimpl.h" +#include "querynodes.h" #include "scalar.h" #include "taoserror.h" #include "tdatablock.h" @@ -185,6 +185,19 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + + SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, 1); + if (nodeType(pParamNode) != QUERY_NODE_VALUE) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)pParamNode; + if (pValue->datum.i < 0 || pValue->datum.i > 100) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } + + pValue->notReserved = true; + if (3 == paraNum) { SNode* pPara3 = nodesListGetNode(pFunc->pParameterList, 2); if (QUERY_NODE_VALUE != nodeType(pPara3) || !validAperventileAlgo((SValueNode*)pPara3)) { @@ -215,7 +228,7 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - SValueNode* pValue = (SValueNode*) pParamNode; + SValueNode* pValue = (SValueNode*)pParamNode; if (pValue->node.resType.type != TSDB_DATA_TYPE_BIGINT) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -224,6 +237,8 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); } + pValue->notReserved = true; + SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; return TSDB_CODE_SUCCESS; @@ -336,7 +351,7 @@ static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -361,7 +376,7 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT }; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -392,7 +407,7 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } @@ -434,7 +449,7 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - uint8_t colType = pCol->resType.type; + uint8_t colType = pCol->resType.type; if (IS_VAR_DATA_TYPE(colType)) { pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; } else { @@ -463,7 +478,7 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - uint8_t colType = pCol->resType.type; + uint8_t colType = pCol->resType.type; if (IS_VAR_DATA_TYPE(colType)) { pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; } else { @@ -500,8 +515,7 @@ static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (QUERY_NODE_COLUMN != nodeType(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "The parameters of UNIQUE can only be columns"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE can only be columns"); } pFunc->node.resType = ((SExprNode*)pPara)->resType; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index d5dc967fed..441648e52b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -72,12 +72,20 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) { char path[PATH_MAX] = {0}; if (tsProcPath == NULL) { path[0] = '.'; + #ifdef WINDOWS + GetModuleFileName(NULL, path, PATH_MAX); + taosDirName(path); + #endif } else { strncpy(path, tsProcPath, strlen(tsProcPath)); taosDirName(path); } #ifdef WINDOWS - strcat(path, "udfd.exe"); + if (strlen(path)==0) { + strcat(path, "udfd.exe"); + } else { + strcat(path, "\\udfd.exe"); + } #else strcat(path, "/udfd"); #endif diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index d5fd574aad..c6d1cc0dbf 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,6 +12,9 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common + PUBLIC nodes + PUBLIC scalar + PUBLIC function ) if (${BUILD_WITH_LUCENE}) diff --git a/source/libs/executor/src/indexoperator.c b/source/libs/index/src/indexFilter.c similarity index 93% rename from source/libs/executor/src/indexoperator.c rename to source/libs/index/src/indexFilter.c index 123c77ce9b..0273867ccf 100644 --- a/source/libs/executor/src/indexoperator.c +++ b/source/libs/index/src/indexFilter.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ -#include "indexoperator.h" -#include "executorimpl.h" #include "index.h" +#include "indexInt.h" #include "nodes.h" +#include "querynodes.h" +#include "scalar.h" #include "tdatablock.h" // clang-format off @@ -69,9 +70,9 @@ typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output) static sif_func_t sifNullFunc = NULL; // typedef struct SIFWalkParm // construct tag filter operator later -static void destroyTagFilterOperatorInfo(void *param) { - STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; -} +// static void destroyTagFilterOperatorInfo(void *param) { +// STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; +//} static void sifFreeParam(SIFParam *param) { if (param == NULL) return; @@ -178,13 +179,13 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { case QUERY_NODE_NODE_LIST: { SNodeListNode *nl = (SNodeListNode *)node; if (LIST_LENGTH(nl->pNodeList) <= 0) { - qError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); + indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->dataType.type)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); - qError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); + indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } break; @@ -194,7 +195,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { case QUERY_NODE_LOGIC_CONDITION: { SIFParam *res = (SIFParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { - qError("no result for node, type:%d, node:%p", nodeType(node), node); + indexError("no result for node, type:%d, node:%p", nodeType(node), node); SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *param = *res; @@ -210,7 +211,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (NULL == node->pLeft || (nParam == 2 && NULL == node->pRight)) { - qError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight); + indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); @@ -232,7 +233,7 @@ static int32_t sifInitParamList(SIFParam **params, SNodeList *nodeList, SIFCtx * int32_t code = 0; SIFParam *tParams = taosMemoryCalloc(nodeList->length, sizeof(SIFParam)); if (tParams == NULL) { - qError("failed to calloc, nodeList: %p", nodeList); + indexError("failed to calloc, nodeList: %p", nodeList); SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -252,7 +253,7 @@ _return: SIF_RET(code); } static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *output) { - qError("index-filter not support buildin function"); + indexError("index-filter not support buildin function"); return TSDB_CODE_QRY_INVALID_INPUT; } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { @@ -390,8 +391,8 @@ _return: static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *output) { if (NULL == node->pParameterList || node->pParameterList->length <= 0) { - qError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, - node->pParameterList ? node->pParameterList->length : 0); + indexError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, + node->pParameterList ? node->pParameterList->length : 0); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -485,7 +486,7 @@ EDealRes sifCalcWalker(SNode *node, void *context) { return sifWalkOper(node, ctx); } - qError("invalid node type for index filter calculating, type:%d", nodeType(node)); + indexError("invalid node type for index filter calculating, type:%d", nodeType(node)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } @@ -509,7 +510,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { SIFCtx ctx = {.code = 0, .noExec = false}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { - qError("index-filter failed to taosHashInit"); + indexError("index-filter failed to taosHashInit"); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -519,7 +520,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (pDst) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { - qError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); + indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } taosArrayAddAll(pDst->result, res->result); @@ -539,7 +540,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { SIFCtx ctx = {.code = 0, .noExec = true}; ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { - qError("index-filter failed to taosHashInit"); + indexError("index-filter failed to taosHashInit"); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -549,7 +550,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { - qError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); + indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *status = res->status; diff --git a/source/libs/executor/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp similarity index 100% rename from source/libs/executor/test/index_executor_tests.cpp rename to source/libs/index/test/index_executor_tests.cpp diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 8019200e76..5774dcaa1d 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -19,6 +19,21 @@ #include "taos.h" #include "taoserror.h" +#define COPY_SCALAR_FIELD(fldname) \ + do { \ + (pDst)->fldname = (pSrc)->fldname; \ + } while (0) + +#define COPY_CHAR_ARRAY_FIELD(fldname) \ + do { \ + strcpy((pDst)->fldname, (pSrc)->fldname); \ + } while (0) + +#define COPY_OBJECT_FIELD(fldname, size) \ + do { \ + memcpy(&((pDst)->fldname), &((pSrc)->fldname), size); \ + } while (0) + #define COPY_CHAR_POINT_FIELD(fldname) \ do { \ if (NULL == (pSrc)->fldname) { \ @@ -70,27 +85,61 @@ } \ } while (0) -static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {} - static SNode* exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { - dataTypeCopy(&pSrc->resType, &pDst->resType); - pDst->pAssociation = NULL; + COPY_OBJECT_FIELD(resType, sizeof(SDataType)); + COPY_CHAR_ARRAY_FIELD(aliasName); + COPY_CHAR_ARRAY_FIELD(userAlias); return (SNode*)pDst; } static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); - pDst->pProjectRef = NULL; + COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(tableType); + COPY_SCALAR_FIELD(colId); + COPY_SCALAR_FIELD(colType); + COPY_CHAR_ARRAY_FIELD(dbName); + COPY_CHAR_ARRAY_FIELD(tableName); + COPY_CHAR_ARRAY_FIELD(tableAlias); + COPY_CHAR_ARRAY_FIELD(colName); + COPY_SCALAR_FIELD(dataBlockId); + COPY_SCALAR_FIELD(slotId); return (SNode*)pDst; } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); COPY_CHAR_POINT_FIELD(literal); + COPY_SCALAR_FIELD(isDuration); + COPY_SCALAR_FIELD(translate); + COPY_SCALAR_FIELD(notReserved); + COPY_SCALAR_FIELD(placeholderNo); + COPY_SCALAR_FIELD(typeData); + COPY_SCALAR_FIELD(unit); if (!pSrc->translate) { return (SNode*)pDst; } switch (pSrc->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + COPY_SCALAR_FIELD(datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + COPY_SCALAR_FIELD(datum.i); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + COPY_SCALAR_FIELD(datum.d); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + COPY_SCALAR_FIELD(datum.u); + break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: @@ -104,7 +153,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: - // todo + case TSDB_DATA_TYPE_MEDIUMBLOB: default: break; } @@ -113,6 +162,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_SCALAR_FIELD(opType); CLONE_NODE_FIELD(pLeft); CLONE_NODE_FIELD(pRight); return (SNode*)pDst; @@ -120,18 +170,27 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_SCALAR_FIELD(condType); CLONE_NODE_LIST_FIELD(pParameterList); return (SNode*)pDst; } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_CHAR_ARRAY_FIELD(functionName); + COPY_SCALAR_FIELD(funcId); + COPY_SCALAR_FIELD(funcType); CLONE_NODE_LIST_FIELD(pParameterList); + COPY_SCALAR_FIELD(udfBufSize); return (SNode*)pDst; } static SNode* tableNodeCopy(const STableNode* pSrc, STableNode* pDst) { COPY_BASE_OBJECT_FIELD(node, exprNodeCopy); + COPY_CHAR_ARRAY_FIELD(dbName); + COPY_CHAR_ARRAY_FIELD(tableName); + COPY_CHAR_ARRAY_FIELD(tableAlias); + COPY_SCALAR_FIELD(precision); return (SNode*)pDst; } @@ -159,6 +218,8 @@ static SNode* realTableNodeCopy(const SRealTableNode* pSrc, SRealTableNode* pDst COPY_BASE_OBJECT_FIELD(table, tableNodeCopy); CLONE_OBJECT_FIELD(pMeta, tableMetaClone); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); + COPY_CHAR_ARRAY_FIELD(qualDbName); + COPY_SCALAR_FIELD(ratio); return (SNode*)pDst; } @@ -170,6 +231,7 @@ static SNode* tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDst static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) { COPY_BASE_OBJECT_FIELD(table, tableNodeCopy); + COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pLeft); CLONE_NODE_FIELD(pRight); CLONE_NODE_FIELD(pOnCond); @@ -177,21 +239,30 @@ static SNode* joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst } static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { + COPY_SCALAR_FIELD(dataBlockId); + COPY_SCALAR_FIELD(slotId); CLONE_NODE_FIELD(pExpr); return (SNode*)pDst; } static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) { + COPY_SCALAR_FIELD(groupingSetType); CLONE_NODE_LIST_FIELD(pParameterList); return (SNode*)pDst; } static SNode* orderByExprNodeCopy(const SOrderByExprNode* pSrc, SOrderByExprNode* pDst) { CLONE_NODE_FIELD(pExpr); + COPY_SCALAR_FIELD(order); + COPY_SCALAR_FIELD(nullOrder); return (SNode*)pDst; } -static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) { return (SNode*)pDst; } +static SNode* limitNodeCopy(const SLimitNode* pSrc, SLimitNode* pDst) { + COPY_SCALAR_FIELD(limit); + COPY_SCALAR_FIELD(offset); + return (SNode*)pDst; +} static SNode* stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNode* pDst) { CLONE_NODE_FIELD(pCol); @@ -215,13 +286,16 @@ static SNode* intervalWindowNodeCopy(const SIntervalWindowNode* pSrc, SIntervalW } static SNode* nodeListNodeCopy(const SNodeListNode* pSrc, SNodeListNode* pDst) { + COPY_OBJECT_FIELD(dataType, sizeof(SDataType)); CLONE_NODE_LIST_FIELD(pNodeList); return (SNode*)pDst; } static SNode* fillNodeCopy(const SFillNode* pSrc, SFillNode* pDst) { + COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pValues); CLONE_NODE_FIELD(pWStartTs); + COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow)); return (SNode*)pDst; } @@ -229,7 +303,7 @@ static SNode* logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pTargets); CLONE_NODE_FIELD(pConditions); CLONE_NODE_LIST_FIELD(pChildren); - pDst->pParent = NULL; + COPY_SCALAR_FIELD(optimizedFlag); return (SNode*)pDst; } @@ -239,12 +313,25 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pScanPseudoCols); CLONE_OBJECT_FIELD(pMeta, tableMetaClone); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); + COPY_SCALAR_FIELD(scanType); + COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2); + COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); + COPY_OBJECT_FIELD(tableName, sizeof(SName)); + COPY_SCALAR_FIELD(showRewrite); + COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); + COPY_SCALAR_FIELD(dataRequired); + COPY_SCALAR_FIELD(interval); + COPY_SCALAR_FIELD(offset); + COPY_SCALAR_FIELD(sliding); + COPY_SCALAR_FIELD(intervalUnit); + COPY_SCALAR_FIELD(slidingUnit); return (SNode*)pDst; } static SNode* logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pOnConditions); return (SNode*)pDst; } @@ -259,32 +346,50 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pProjections); + COPY_CHAR_ARRAY_FIELD(stmtName); + COPY_SCALAR_FIELD(limit); + COPY_SCALAR_FIELD(offset); + COPY_SCALAR_FIELD(slimit); + COPY_SCALAR_FIELD(soffset); return (SNode*)pDst; } static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); - pDst->pDataBlocks = NULL; - pDst->pVgDataBlocks = NULL; + COPY_SCALAR_FIELD(msgType); return (SNode*)pDst; } static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(srcGroupId); + COPY_SCALAR_FIELD(precision); return (SNode*)pDst; } static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(winType); CLONE_NODE_LIST_FIELD(pFuncs); + COPY_SCALAR_FIELD(interval); + COPY_SCALAR_FIELD(offset); + COPY_SCALAR_FIELD(sliding); + COPY_SCALAR_FIELD(intervalUnit); + COPY_SCALAR_FIELD(slidingUnit); + COPY_SCALAR_FIELD(sessionGap); CLONE_NODE_FIELD(pTspk); + CLONE_NODE_FIELD(pStateExpr); + COPY_SCALAR_FIELD(triggerType); + COPY_SCALAR_FIELD(watermark); return (SNode*)pDst; } static SNode* logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(mode); CLONE_NODE_FIELD(pWStartTs); CLONE_NODE_FIELD(pValues); + COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow)); return (SNode*)pDst; } @@ -301,28 +406,41 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi } static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) { + COPY_OBJECT_FIELD(id, sizeof(SSubplanId)); CLONE_NODE_FIELD(pNode); - pDst->pChildren = NULL; - pDst->pParents = NULL; - pDst->pVgroupList = NULL; + COPY_SCALAR_FIELD(subplanType); + COPY_SCALAR_FIELD(level); + COPY_SCALAR_FIELD(splitFlag); return (SNode*)pDst; } static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) { + COPY_SCALAR_FIELD(dataBlockId); CLONE_NODE_LIST_FIELD(pSlots); + COPY_SCALAR_FIELD(totalRowSize); + COPY_SCALAR_FIELD(outputRowSize); + COPY_SCALAR_FIELD(precision); return (SNode*)pDst; } static SNode* slotDescCopy(const SSlotDescNode* pSrc, SSlotDescNode* pDst) { - dataTypeCopy(&pSrc->dataType, &pDst->dataType); + COPY_SCALAR_FIELD(slotId); + COPY_OBJECT_FIELD(dataType, sizeof(SDataType)); + COPY_SCALAR_FIELD(reserve); + COPY_SCALAR_FIELD(output); + COPY_SCALAR_FIELD(tag); return (SNode*)pDst; } static SNode* downstreamSourceCopy(const SDownstreamSourceNode* pSrc, SDownstreamSourceNode* pDst) { + COPY_OBJECT_FIELD(addr, sizeof(SQueryNodeAddr)); + COPY_SCALAR_FIELD(taskId); + COPY_SCALAR_FIELD(schedId); return (SNode*)pDst; } static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { + COPY_SCALAR_FIELD(isDistinct); CLONE_NODE_LIST_FIELD(pProjectionList); CLONE_NODE_FIELD(pFromTable); CLONE_NODE_FIELD(pWhere); @@ -333,6 +451,12 @@ static SNode* selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { CLONE_NODE_LIST_FIELD(pOrderByList); CLONE_NODE_FIELD(pLimit); CLONE_NODE_FIELD(pLimit); + COPY_CHAR_ARRAY_FIELD(stmtName); + COPY_SCALAR_FIELD(precision); + COPY_SCALAR_FIELD(isEmptyResult); + COPY_SCALAR_FIELD(isTimeOrderQuery); + COPY_SCALAR_FIELD(hasAggFuncs); + COPY_SCALAR_FIELD(hasRepeatScanFuncs); return (SNode*)pDst; } @@ -345,7 +469,6 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - memcpy(pDst, pNode, nodesNodeSize(nodeType(pNode))); switch (nodeType(pNode)) { case QUERY_NODE_COLUMN: return columnNodeCopy((const SColumnNode*)pNode, (SColumnNode*)pDst); @@ -387,6 +510,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return slotDescCopy((const SSlotDescNode*)pNode, (SSlotDescNode*)pDst); case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceCopy((const SDownstreamSourceNode*)pNode, (SDownstreamSourceNode*)pDst); + case QUERY_NODE_LEFT_VALUE: + return pDst; case QUERY_NODE_SELECT_STMT: return selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst); case QUERY_NODE_LOGIC_PLAN_SCAN: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0e8f530b0e..9e3608ec1a 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -78,6 +78,8 @@ const char* nodesNodeName(ENodeType type) { return "TableOptions"; case QUERY_NODE_INDEX_OPTIONS: return "IndexOptions"; + case QUERY_NODE_LEFT_VALUE: + return "LeftValue"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: @@ -2175,7 +2177,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { code = TSDB_CODE_OUT_OF_MEMORY; break; } - varDataSetLen(pNode->datum.p, pNode->node.resType.bytes); + varDataSetLen(pNode->datum.p, pNode->node.resType.bytes - VARSTR_HEADER_SIZE); if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) { char* buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1); if (NULL == buf) { @@ -3019,6 +3021,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceNodeToJson(pObj, pJson); + case QUERY_NODE_LEFT_VALUE: + return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to serialize. case QUERY_NODE_SET_OPERATOR: return setOperatorToJson(pObj, pJson); case QUERY_NODE_SELECT_STMT: @@ -3130,6 +3134,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToSlotDescNode(pJson, pObj); case QUERY_NODE_DOWNSTREAM_SOURCE: return jsonToDownstreamSourceNode(pJson, pObj); + case QUERY_NODE_LEFT_VALUE: + return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to deserialize. case QUERY_NODE_SET_OPERATOR: return jsonToSetOperator(pJson, pObj); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 578a1aac48..3f7003dfa3 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -79,6 +79,8 @@ int32_t nodesNodeSize(ENodeType type) { return sizeof(SStreamOptions); case QUERY_NODE_TOPIC_OPTIONS: return sizeof(STopicOptions); + case QUERY_NODE_LEFT_VALUE: + return sizeof(SLeftValueNode); case QUERY_NODE_SET_OPERATOR: return sizeof(SSetOperator); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index eddf126aec..3a605dd779 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -565,132 +565,144 @@ static int32_t parseTimeFromValueNode(SValueNode* pVal) { } static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) { - uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision); - pVal->node.resType.precision = precision; - if (pVal->placeholderNo > 0) { - return DEAL_RES_CONTINUE; - } - if (pVal->isDuration) { - if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) != - TSDB_CODE_SUCCESS) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - } - *(int64_t*)&pVal->typeData = pVal->datum.i; - } else { - switch (targetDt.type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); - *(bool*)&pVal->typeData = pVal->datum.b; - break; - case TSDB_DATA_TYPE_TINYINT: { - char* endPtr = NULL; - pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); - *(int8_t*)&pVal->typeData = pVal->datum.i; - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - char* endPtr = NULL; - pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); - *(int16_t*)&pVal->typeData = pVal->datum.i; - break; - } - case TSDB_DATA_TYPE_INT: { - char* endPtr = NULL; - pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); - *(int32_t*)&pVal->typeData = pVal->datum.i; - break; - } - case TSDB_DATA_TYPE_BIGINT: { - char* endPtr = NULL; - pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); - *(int64_t*)&pVal->typeData = pVal->datum.i; - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - char* endPtr = NULL; - pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); - *(uint8_t*)&pVal->typeData = pVal->datum.u; - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - char* endPtr = NULL; - pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); - *(uint16_t*)&pVal->typeData = pVal->datum.u; - break; - } - case TSDB_DATA_TYPE_UINT: { - char* endPtr = NULL; - pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); - *(uint32_t*)&pVal->typeData = pVal->datum.u; - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - char* endPtr = NULL; - pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); - *(uint64_t*)&pVal->typeData = pVal->datum.u; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - char* endPtr = NULL; - pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); - *(float*)&pVal->typeData = pVal->datum.d; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - char* endPtr = NULL; - pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); - *(double*)&pVal->typeData = pVal->datum.d; - break; - } - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY: { - pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + VARSTR_HEADER_SIZE + 1); - if (NULL == pVal->datum.p) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); - } - varDataSetLen(pVal->datum.p, targetDt.bytes); - strncpy(varDataVal(pVal->datum.p), pVal->literal, targetDt.bytes); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: { - if (TSDB_CODE_SUCCESS != parseTimeFromValueNode(pVal)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - } - *(int64_t*)&pVal->typeData = pVal->datum.i; - break; - } - case TSDB_DATA_TYPE_NCHAR: { - int32_t bytes = targetDt.bytes * TSDB_NCHAR_SIZE; - pVal->datum.p = taosMemoryCalloc(1, bytes + VARSTR_HEADER_SIZE + 1); - if (NULL == pVal->datum.p) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); - ; - } + uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision); + pVal->node.resType.precision = precision; + if (pVal->placeholderNo > 0) { + return DEAL_RES_CONTINUE; + } + if (pVal->isDuration) { + if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) != + TSDB_CODE_SUCCESS) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + *(int64_t*)&pVal->typeData = pVal->datum.i; + } else { + switch (targetDt.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); + *(bool*)&pVal->typeData = pVal->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: { + char* endPtr = NULL; + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); + *(int8_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + char* endPtr = NULL; + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); + *(int16_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_INT: { + char* endPtr = NULL; + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); + *(int32_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_BIGINT: { + char* endPtr = NULL; + pVal->datum.i = taosStr2Int64(pVal->literal, &endPtr, 10); + *(int64_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + char* endPtr = NULL; + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); + *(uint8_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + char* endPtr = NULL; + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); + *(uint16_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_UINT: { + char* endPtr = NULL; + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); + *(uint32_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + char* endPtr = NULL; + pVal->datum.u = taosStr2UInt64(pVal->literal, &endPtr, 10); + *(uint64_t*)&pVal->typeData = pVal->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + char* endPtr = NULL; + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); + *(float*)&pVal->typeData = pVal->datum.d; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* endPtr = NULL; + pVal->datum.d = taosStr2Double(pVal->literal, &endPtr); + *(double*)&pVal->typeData = pVal->datum.d; + break; + } + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: { + pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); + if (NULL == pVal->datum.p) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + } + int32_t len = TMIN(targetDt.bytes - VARSTR_HEADER_SIZE, pVal->node.resType.bytes); + varDataSetLen(pVal->datum.p, len); + strncpy(varDataVal(pVal->datum.p), pVal->literal, len); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: { + if (TSDB_CODE_SUCCESS != parseTimeFromValueNode(pVal)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + *(int64_t*)&pVal->typeData = pVal->datum.i; + break; + } + case TSDB_DATA_TYPE_NCHAR: { + pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); + if (NULL == pVal->datum.p) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); + ; + } - int32_t output = 0; - if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), bytes, - &output)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - } - varDataSetLen(pVal->datum.p, output); - break; - } - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); - default: - break; - } - } - pVal->node.resType = targetDt; - pVal->translate = true; - return DEAL_RES_CONTINUE; + int32_t len = 0; + if (!taosMbsToUcs4(pVal->literal, pVal->node.resType.bytes, (TdUcs4*)varDataVal(pVal->datum.p), + targetDt.bytes - VARSTR_HEADER_SIZE, &len)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } + varDataSetLen(pVal->datum.p, len); + break; + } + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + default: + break; + } + } + pVal->node.resType = targetDt; + pVal->translate = true; + return DEAL_RES_CONTINUE; +} + +static int32_t calcTypeBytes(SDataType dt) { + if (TSDB_DATA_TYPE_BINARY == dt.type) { + return dt.bytes + VARSTR_HEADER_SIZE; + } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { + return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else { + return dt.bytes; + } } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { - return translateValueImpl(pCxt, pVal, pVal->node.resType); + SDataType dt = pVal->node.resType; + dt.bytes = calcTypeBytes(dt); + return translateValueImpl(pCxt, pVal, dt); } static bool isMultiResFunc(SNode* pNode) { @@ -2341,16 +2353,6 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm return buildCmdMsg(pCxt, TDMT_MND_ALTER_DB, (FSerializeFunc)tSerializeSAlterDbReq, &alterReq); } -static int32_t calcTypeBytes(SDataType dt) { - if (TSDB_DATA_TYPE_BINARY == dt.type) { - return dt.bytes + VARSTR_HEADER_SIZE; - } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { - return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else { - return dt.bytes; - } -} - static int32_t columnDefNodeToField(SNodeList* pList, SArray** pArray) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); SNode* pNode; @@ -4089,19 +4091,9 @@ static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* p return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); } -static int32_t colDataBytesToValueDataBytes(uint8_t type, int32_t bytes) { - if (TSDB_DATA_TYPE_VARCHAR == type || TSDB_DATA_TYPE_BINARY == type || TSDB_DATA_TYPE_VARBINARY == type) { - return bytes - VARSTR_HEADER_SIZE; - } else if (TSDB_DATA_TYPE_NCHAR == type) { - return (bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; - } - return bytes; -} - static SDataType schemaToDataType(SSchema* pSchema) { - SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0}; - dt.bytes = colDataBytesToValueDataBytes(pSchema->type, pSchema->bytes); - return dt; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes, .precision = 0, .scale = 0}; + return dt; } static int32_t translateTagVal(STranslateContext* pCxt, SSchema* pSchema, SNode* pNode, SValueNode** pVal) { diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index cc0dded570..784586dfb2 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -204,7 +204,7 @@ TEST_F(ParserInitialATest, alterTable) { } }; - auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, const uint8_t* pNewVal, uint32_t bytes) { + auto setAlterTagFunc = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) { memset(&expect, 0, sizeof(SVAlterTbReq)); expect.tbName = strdup(pTbname); expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; @@ -215,7 +215,7 @@ TEST_F(ParserInitialATest, alterTable) { expect.pTagVal = pNewVal; }; - auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, const char* pComment = nullptr) { + auto setAlterOptionsFunc = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) { memset(&expect, 0, sizeof(SVAlterTbReq)); expect.tbName = strdup(pTbname); expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; @@ -240,7 +240,7 @@ TEST_F(ParserInitialATest, alterTable) { void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); SVAlterTbReq req = {0}; SDecoder coder = {0}; - tDecoderInit(&coder, (const uint8_t*)pBuf, pVgData->size); + tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); @@ -274,7 +274,7 @@ TEST_F(ParserInitialATest, alterTable) { setAlterOptionsFunc("t1", 10, nullptr); run("ALTER TABLE t1 TTL 10"); - setAlterOptionsFunc("t1", -1, "test"); + setAlterOptionsFunc("t1", -1, (char*)"test"); run("ALTER TABLE t1 COMMENT 'test'"); setAlterColFunc("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); @@ -290,7 +290,7 @@ TEST_F(ParserInitialATest, alterTable) { run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); int32_t val = 10; - setAlterTagFunc("st1s1", "tag1", (const uint8_t*)&val, sizeof(val)); + setAlterTagFunc("st1s1", "tag1", (uint8_t*)&val, sizeof(val)); run("ALTER TABLE st1s1 SET TAG tag1=10"); // todo diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index affe9ef2f6..cf9be2c81c 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -599,14 +599,17 @@ typedef struct SRewritePrecalcExprsCxt { static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { SNode* pExpr = nodesCloneNode(*pNode); if (NULL == pExpr) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; nodesDestroyNode(pExpr); return DEAL_RES_ERROR; } SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; nodesDestroyNode(pExpr); return DEAL_RES_ERROR; } @@ -624,16 +627,45 @@ static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) return DEAL_RES_IGNORE_CHILD; } +static int32_t rewriteValueToOperator(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { + SOperatorNode* pOper = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + if (NULL == pOper) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pOper->pLeft = nodesMakeNode(QUERY_NODE_LEFT_VALUE); + if (NULL == pOper->pLeft) { + nodesDestroyNode(pOper); + return TSDB_CODE_OUT_OF_MEMORY; + } + SValueNode* pVal = (SValueNode*)*pNode; + pOper->node.resType = pVal->node.resType; + strcpy(pOper->node.aliasName, pVal->node.aliasName); + pOper->opType = OP_TYPE_ASSIGN; + pOper->pRight = *pNode; + *pNode = (SNode*)pOper; + return TSDB_CODE_SUCCESS; +} + static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext; switch (nodeType(*pNode)) { + case QUERY_NODE_VALUE: { + if (((SValueNode*)*pNode)->notReserved) { + break; + } + pCxt->errCode = rewriteValueToOperator(pCxt, pNode); + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + return DEAL_RES_ERROR; + } + return collectAndRewrite(pCxt, pNode); + } case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: { - return collectAndRewrite(pContext, pNode); + return collectAndRewrite(pCxt, pNode); } case QUERY_NODE_FUNCTION: { if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) { - return collectAndRewrite(pContext, pNode); + return collectAndRewrite(pCxt, pNode); } } default: @@ -677,9 +709,8 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } SRewritePrecalcExprsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs}; nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); - if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { - nodesDestroyList(cxt.pPrecalcExprs); - *pPrecalcExprs = NULL; + if (0 == LIST_LENGTH(cxt.pPrecalcExprs) || TSDB_CODE_SUCCESS != cxt.errCode) { + DESTORY_LIST(*pPrecalcExprs); } return cxt.errCode; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index a17d8cd850..4b84079f7b 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -50,4 +50,6 @@ TEST_F(PlanBasicTest, func) { run("SELECT DIFF(c1) FROM t1"); run("SELECT PERCENTILE(c1, 60) FROM t1"); + + run("SELECT TOP(c1, 60) FROM t1"); } diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 9ca1001f4c..cf51603470 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -49,6 +49,8 @@ TEST_F(PlanGroupByTest, aggFunc) { run("SELECT LAST(*), FIRST(*) FROM t1"); run("SELECT LAST(*), FIRST(*) FROM t1 GROUP BY c1"); + + run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } TEST_F(PlanGroupByTest, selectFunc) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 6e184fec72..86bacc9220 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -14,6 +14,7 @@ */ #include "planTestUtil.h" +#include #include #include diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 49ed3ab48b..fb03eaefa4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -182,6 +182,11 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { + case QUERY_NODE_LEFT_VALUE: { + SSDataBlock* pb = taosArrayGetP(ctx->pBlockList, 0); + param->numOfRows = pb->info.rows; + break; + } case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; @@ -845,7 +850,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { } EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)|| QUERY_NODE_LEFT_VALUE == nodeType(pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 45742189d5..12496eec55 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -824,7 +824,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } //for constant conversion, need to set proper length of pOutput description if (len < outputLen) { - pOutput->columnData->info.bytes = len; + pOutput->columnData->info.bytes = len + VARSTR_HEADER_SIZE; } break; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 145ed69a77..0fb3712c30 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1333,6 +1333,22 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO doReleaseVec(pLeftCol, leftConvert); } +void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + pOut->numOfRows = pLeft->numOfRows; + + if (colDataIsNull_s(pRight->columnData, 0)) { + for (int32_t i = 0; i < pOut->numOfRows; ++i) { + colDataAppend(pOutputCol, i, NULL, true); + } + } else { + for (int32_t i = 0; i < pOut->numOfRows; ++i) { + colDataAppend(pOutputCol, i, colDataGetData(pRight->columnData, 0), false); + } + } +} + void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { #if 0 int32_t len = pLeft->bytes + pRight->bytes; @@ -1691,6 +1707,8 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorMathRemainder; case OP_TYPE_MINUS: return vectorMathMinus; + case OP_TYPE_ASSIGN: + return vectorAssign; case OP_TYPE_GREATER_THAN: return vectorGreater; case OP_TYPE_GREATER_EQUAL: diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index be92de774b..9e0878e118 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -132,7 +132,7 @@ typedef struct SSchLevel { int32_t taskSucceed; int32_t taskNum; int32_t taskLaunchedNum; - SHashObj *flowCtrl; // key is ep, element is SSchFlowControl + int32_t taskDoneNum; SArray *subTasks; // Element is SQueryTask } SSchLevel; @@ -175,11 +175,13 @@ typedef struct SSchJob { SArray *levels; // starting from 0. SArray SNodeList *subPlans; // subplan pointer copied from DAG, no need to free it in scheduler + SArray *dataSrcTasks; // SArray int32_t levelIdx; SEpSet dataSrcEps; SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* + SHashObj *flowCtrl; // key is ep, element is SSchFlowControl SExplainCtx *explainCtx; int8_t status; @@ -200,7 +202,7 @@ typedef struct SSchJob { extern SSchedulerMgmt schMgmt; -#define SCH_TASK_READY_TO_LUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) +#define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0) @@ -223,7 +225,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) -#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEAF_TASK(_job, _task) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) +#define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_SET_JOB_TYPE(_job, type) (_job)->attr.queryJob = ((type) != SUBPLAN_TYPE_MODIFY) #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) @@ -261,7 +263,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); int32_t schReleaseJob(int64_t refId); -void schFreeFlowCtrl(SSchLevel *pLevel); +void schFreeFlowCtrl(SSchJob *pJob); int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 993521da87..fbcca403bb 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -19,13 +19,13 @@ #include "catalog.h" #include "tref.h" -void schFreeFlowCtrl(SSchLevel *pLevel) { - if (NULL == pLevel->flowCtrl) { +void schFreeFlowCtrl(SSchJob *pJob) { + if (NULL == pJob->flowCtrl) { return; } SSchFlowControl *ctrl = NULL; - void *pIter = taosHashIterate(pLevel->flowCtrl, NULL); + void *pIter = taosHashIterate(pJob->flowCtrl, NULL); while (pIter) { ctrl = (SSchFlowControl *)pIter; @@ -33,11 +33,11 @@ void schFreeFlowCtrl(SSchLevel *pLevel) { taosArrayDestroy(ctrl->taskList); } - pIter = taosHashIterate(pLevel->flowCtrl, pIter); + pIter = taosHashIterate(pJob->flowCtrl, pIter); } - taosHashCleanup(pLevel->flowCtrl); - pLevel->flowCtrl = NULL; + taosHashCleanup(pJob->flowCtrl); + pJob->flowCtrl = NULL; } int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { @@ -47,9 +47,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { } int32_t sum = 0; - - for (int32_t i = 0; i < pLevel->taskNum; ++i) { - SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); + int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); + for (int32_t i = 0; i < taskNum; ++i) { + SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); sum += pTask->plan->execNodeStat.tableNum; } @@ -59,9 +59,9 @@ int32_t schCheckJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { return TSDB_CODE_SUCCESS; } - pLevel->flowCtrl = taosHashInit(pLevel->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - if (NULL == pLevel->flowCtrl) { - SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pLevel->taskNum); + pJob->flowCtrl = taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pJob->flowCtrl) { + SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -78,7 +78,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp)); + ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); @@ -110,11 +110,11 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); do { - ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp)); + ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SSchFlowControl nctrl = {.tableNumSum = pTask->plan->execNodeStat.tableNum, .execTaskNum = 1}; - code = taosHashPut(pLevel->flowCtrl, ep, sizeof(SEp), &nctrl, sizeof(nctrl)); + code = taosHashPut(pJob->flowCtrl, ep, sizeof(SEp), &nctrl, sizeof(nctrl)); if (code) { if (HASH_NODE_EXIST(code)) { continue; @@ -273,10 +273,9 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask)); - SSchLevel *pLevel = pTask->level; SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); - SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pLevel->flowCtrl, ep, sizeof(SEp)); + SSchFlowControl *ctrl = (SSchFlowControl *)taosHashGet(pJob->flowCtrl, ep, sizeof(SEp)); if (NULL == ctrl) { SCH_TASK_ELOG("taosHashGet node from flowCtrl failed, fqdn:%s, port:%d", ep->fqdn, ep->port); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index dcd87557aa..7a75c00e6e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -391,6 +391,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + SCH_TASK_DLOG("children info, the %d child TID %" PRIx64, n, (*childTask)->taskId); } if (parentNum > 0) { @@ -423,6 +425,8 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + + SCH_TASK_DLOG("parents info, the %d parent TID %" PRIx64, n, (*parentTask)->taskId); } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); @@ -464,6 +468,17 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad return TSDB_CODE_SUCCESS; } +int32_t schRecordQueryDataSrc(SSchJob *pJob, SSchTask *pTask) { + if (!SCH_IS_DATA_SRC_QRY_TASK(pTask)) { + return TSDB_CODE_SUCCESS; + } + + taosArrayPush(pJob->dataSrcTasks, &pTask); + + return TSDB_CODE_SUCCESS; +} + + int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { int32_t code = 0; pJob->queryId = pDag->queryId; @@ -473,6 +488,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + pJob->dataSrcTasks = taosArrayInit(pDag->numOfSubplans, POINTER_BYTES); + if (NULL == pJob->dataSrcTasks) { + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + int32_t levelNum = (int32_t)LIST_LENGTH(pDag->pSubplans); if (levelNum <= 0) { SCH_JOB_ELOG("invalid level num:%d", levelNum); @@ -551,6 +571,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_ERR_JRET(schRecordQueryDataSrc(pJob, p)); + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -629,6 +651,17 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) { + int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId)); + if (code) { + SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); if (0 != code) { @@ -774,6 +807,9 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask)); + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); + if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { SCH_ERR_RET(schDecTaskFlowQuota(pJob, pTask)); SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask)); @@ -947,6 +983,32 @@ _return: SCH_RET(schProcessOnJobFailure(pJob, errCode)); } +int32_t schLaunchNextLevelTasks(SSchJob *pJob, SSchTask *pTask) { + if (!SCH_IS_QUERY_JOB(pJob)) { + return TSDB_CODE_SUCCESS; + } + + SSchLevel *pLevel = pTask->level; + int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); + if (doneNum == pLevel->taskNum) { + pJob->levelIdx--; + + pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); + for (int32_t i = 0; i < pLevel->taskNum; ++i) { + SSchTask *pTask = taosArrayGet(pLevel->subTasks, i); + + if (pTask->children && taosArrayGetSize(pTask->children) > 0) { + continue; + } + + SCH_ERR_RET(schLaunchTask(pJob, pTask)); + } + } + + return TSDB_CODE_SUCCESS; +} + + // Note: no more task error processing, handled in function internal int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { bool moved = false; @@ -1015,11 +1077,13 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { qSetSubplanExecutionNode(par->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { - SCH_ERR_RET(schLaunchTaskImpl(pJob, par)); + if (SCH_TASK_READY_FOR_LAUNCH(readyNum, par)) { + SCH_ERR_RET(schLaunchTask(pJob, par)); } } + SCH_ERR_RET(schLaunchNextLevelTasks(pJob, pTask)); + return TSDB_CODE_SUCCESS; _return: @@ -2400,8 +2464,6 @@ void schFreeJobImpl(void *job) { for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); - schFreeFlowCtrl(pLevel); - int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for (int32_t j = 0; j < numOfTasks; ++j) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); @@ -2411,12 +2473,15 @@ void schFreeJobImpl(void *job) { taosArrayDestroy(pLevel->subTasks); } + schFreeFlowCtrl(pJob); + taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->failTasks); taosHashCleanup(pJob->succTasks); taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->nodeList); + taosArrayDestroy(pJob->dataSrcTasks); qExplainFreeCtx(pJob->explainCtx); diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index 66a661481e..0acec0e4e6 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -134,7 +134,7 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) { } static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { - void* exec = pTask->exec.runners[0].executor; + void* exec = pTask->exec.executor; // set input if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { @@ -171,12 +171,12 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } // TODO: handle version -int32_t streamTaskExec2(SStreamTask* pTask, SMsgCb* pMsgCb) { +int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) return -1; while (1) { int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - void* exec = pTask->exec.runners[0].executor; + void* exec = pTask->exec.executor; if (execStatus == TASK_STATUS__IDLE) { // first run, from qall, handle failure from last exec while (1) { @@ -278,7 +278,7 @@ FAIL: return -1; } -int32_t streamTaskSink(SStreamTask* pTask, SMsgCb* pMsgCb) { +int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) { bool firstRun = 1; while (1) { SStreamDataBlock* pBlock = NULL; @@ -407,7 +407,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* return 0; } -int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { +int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { // 1. handle input streamTaskEnqueue(pTask, pReq, pRsp); @@ -415,172 +415,42 @@ int32_t streamTaskProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStream // 2.1. idle: exec // 2.2. executing: return // 2.3. closing: keep trying - streamTaskExec2(pTask, pMsgCb); + streamExec(pTask, pMsgCb); // 3. handle output // 3.1 check and set status // 3.2 dispatch / sink - streamTaskSink(pTask, pMsgCb); + streamSink(pTask, pMsgCb); return 0; } -int32_t streamTaskProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) { +int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) { atomic_store_8(&pTask->inputStatus, pRsp->inputStatus); if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { // TODO: init recover timer } // continue dispatch - streamTaskSink(pTask, pMsgCb); + streamSink(pTask, pMsgCb); return 0; } int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) { - streamTaskExec2(pTask, pMsgCb); - streamTaskSink(pTask, pMsgCb); + streamExec(pTask, pMsgCb); + streamSink(pTask, pMsgCb); return 0; } -int32_t streamTaskProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) { +int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) { // return 0; } -int32_t streamTaskProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) { +int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) { // return 0; } -int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId) { - SArray* pRes = NULL; - // source - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK && pTask->sourceType != TASK_SOURCE__SCAN) return 0; - - // exec - if (pTask->execType != TASK_EXEC__NONE) { - ASSERT(workId < pTask->exec.numOfRunners); - void* exec = pTask->exec.runners[workId].executor; - pRes = taosArrayInit(0, sizeof(SSDataBlock)); - if (pRes == NULL) { - return -1; - } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - qSetStreamInput(exec, input, inputType); - while (1) { - SSDataBlock* output; - uint64_t ts; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, output); - } - } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) { - const SArray* blocks = (const SArray*)input; - /*int32_t sz = taosArrayGetSize(blocks);*/ - /*for (int32_t i = 0; i < sz; i++) {*/ - /*SSDataBlock* pBlock = taosArrayGet(blocks, i);*/ - /*qSetStreamInput(exec, pBlock, inputType);*/ - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK); - while (1) { - SSDataBlock* output; - uint64_t ts; - if (qExecTask(exec, &output, &ts) < 0) { - ASSERT(false); - } - if (output == NULL) { - break; - } - taosArrayPush(pRes, output); - } - /*}*/ - } else { - ASSERT(0); - } - } else { - ASSERT(inputType == STREAM_DATA_TYPE_SSDATA_BLOCK); - pRes = (SArray*)input; - } - - if (pRes == NULL || taosArrayGetSize(pRes) == 0) return 0; - - // sink - if (pTask->sinkType == TASK_SINK__TABLE) { - // blockDebugShowData(pRes); - pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes); - } else if (pTask->sinkType == TASK_SINK__SMA) { - pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes); - // - } else if (pTask->sinkType == TASK_SINK__FETCH) { - // - } else { - ASSERT(pTask->sinkType == TASK_SINK__NONE); - } - - // dispatch - - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - SRpcMsg dispatchMsg = {0}; - if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) { - ASSERT(0); - return -1; - } - - int32_t qType; - if (pTask->dispatchMsgType == TDMT_VND_TASK_PIPE_EXEC || pTask->dispatchMsgType == TDMT_SND_TASK_PIPE_EXEC) { - qType = FETCH_QUEUE; - } else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC || - pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) { - qType = MERGE_QUEUE; - } else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) { - qType = WRITE_QUEUE; - } else { - ASSERT(0); - } - tmsgPutToQueue(pMsgCb, qType, &dispatchMsg); - - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { - SRpcMsg dispatchMsg = {0}; - SEpSet* pEpSet = NULL; - if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) { - ASSERT(0); - return -1; - } - - tmsgSendReq(pEpSet, &dispatchMsg); - - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - if (pShuffleRes == NULL) { - return -1; - } - - int32_t sz = taosArrayGetSize(pRes); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pRes, i); - SArray* pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t)); - if (pArray == NULL) { - pArray = taosArrayInit(0, sizeof(SSDataBlock)); - if (pArray == NULL) { - return -1; - } - taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*)); - } - taosArrayPush(pArray, pDataBlock); - } - - if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) { - return -1; - } - - } else { - ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); - } - return 0; -} - int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->streamId); @@ -607,20 +477,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId) { pTask->streamId = streamId; pTask->status = TASK_STATUS__IDLE; - pTask->inputQ = taosOpenQueue(); - pTask->outputQ = taosOpenQueue(); - pTask->inputQAll = taosAllocateQall(); - pTask->outputQAll = taosAllocateQall(); - if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL) - goto FAIL; return pTask; -FAIL: - if (pTask->inputQ) taosCloseQueue(pTask->inputQ); - if (pTask->outputQ) taosCloseQueue(pTask->outputQ); - if (pTask->inputQAll) taosFreeQall(pTask->inputQAll); - if (pTask->outputQAll) taosFreeQall(pTask->outputQAll); - if (pTask) taosMemoryFree(pTask); - return NULL; } int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { @@ -722,11 +579,7 @@ void tFreeSStreamTask(SStreamTask* pTask) { taosCloseQueue(pTask->outputQ); // TODO if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); - for (int32_t i = 0; i < pTask->exec.numOfRunners; i++) { - qDestroyTask(pTask->exec.runners[i].executor); - } - taosMemoryFree(pTask->exec.runners); - /*taosMemoryFree(pTask->executor);*/ + qDestroyTask(pTask->exec.executor); taosMemoryFree(pTask); } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index efefcbb3e7..04a989439a 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -411,7 +411,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) { } uint32_t len; char* data = NULL; - if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) { + if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) { return NULL; } assert(len = pMsg->dataLen); @@ -670,7 +670,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) { } uint32_t len; char* data = NULL; - if (tDecodeBinary(&decoder, (const uint8_t**)(&data), &len) < 0) { + if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) { return NULL; } assert(len = pMsg->dataLen); diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 90b8e9dd8a..b6e131d4cc 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -18,14 +18,16 @@ if(USE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY) endif () if(BUILD_ADDR2LINE) - target_include_directories( - os - PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf" - ) + if(NOT TD_WINDOWS) + target_include_directories( + os + PUBLIC "${TD_SOURCE_DIR}/contrib/libdwarf/src/lib/libdwarf" + ) + target_link_libraries( + os PUBLIC addr2line dl z + ) + endif() add_definitions(-DUSE_ADDR2LINE) - target_link_libraries( - os PUBLIC addr2line dl z - ) endif () if(CHECK_STR2INT_ERROR) add_definitions(-DTD_CHECK_STR_TO_INT_ERROR) diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 72654d0084..c4b7c9386e 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -91,7 +91,12 @@ void taosRemoveDir(const char *dirname) { bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { + if (taosDirExist(dirname)) return 0; +#ifdef WINDOWS + int32_t code = _mkdir(dirname, 0755); +#else int32_t code = mkdir(dirname, 0755); +#endif if (code < 0 && errno == EEXIST) { return 0; } @@ -101,36 +106,48 @@ int32_t taosMkDir(const char *dirname) { int32_t taosMulMkDir(const char *dirname) { if (dirname == NULL) return -1; - char * temp = strdup(dirname); + char temp[1024]; +#ifdef WINDOWS + taosRealPath(dirname, temp, sizeof(temp)); +#else + strcpy(temp, dirname); +#endif char * pos = temp; int32_t code = 0; - if (strncmp(temp, "/", 1) == 0) { + if (taosDirExist(temp)) return code; + + if (strncmp(temp, TD_DIRSEP, 1) == 0) { pos += 1; - } else if (strncmp(temp, "./", 2) == 0) { + } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) { pos += 2; } for (; *pos != '\0'; pos++) { - if (*pos == '/') { + if (*pos == TD_DIRSEP[0]) { *pos = '\0'; + #ifdef WINDOWS + code = _mkdir(temp, 0755); + #else code = mkdir(temp, 0755); + #endif if (code < 0 && errno != EEXIST) { - free(temp); return code; } - *pos = '/'; + *pos = TD_DIRSEP[0]; } } - if (*(pos - 1) != '/') { + if (*(pos - 1) != TD_DIRSEP[0]) { + #ifdef WINDOWS + code = _mkdir(temp, 0755); + #else code = mkdir(temp, 0755); + #endif if (code < 0 && errno != EEXIST) { - free(temp); return code; } } - free(temp); // int32_t code = mkdir(dirname, 0755); if (code < 0 && errno == EEXIST) { @@ -233,7 +250,13 @@ char *taosDirName(char *name) { _splitpath(name, Drive1, Dir1, NULL, NULL); size_t dirNameLen = strlen(Drive1) + strlen(Dir1); if (dirNameLen > 0) { - name[dirNameLen] = 0; + if (name[dirNameLen - 1] == '/' || name[dirNameLen - 1] == '\\') { + name[dirNameLen - 1] = 0; + } else { + name[dirNameLen] = 0; + } + } else { + name[0] = 0; } return name; #else diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index aa64e65638..e08b668163 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -109,8 +109,11 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha int64_t taosCopyFile(const char *from, const char *to) { #ifdef WINDOWS - assert(0); - return -1; + if (CopyFile(from, to, 0)) { + return 1; + } else { + return -1; + } #else char buffer[4096]; int64_t size = 0; @@ -236,7 +239,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } -TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { +TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { int fd = -1; FILE *fp = NULL; if (tdFileOptions & TD_FILE_STREAM) { @@ -343,7 +346,11 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { char *tbuf = (char *)buf; while (leftbytes > 0) { + #ifdef WINDOWS + readbytes = _read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); + #else readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); + #endif if (readbytes < 0) { if (errno == EINTR) { continue; @@ -379,10 +386,10 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #endif assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS - size_t pos = lseek(pFile->fd, 0, SEEK_CUR); - lseek(pFile->fd, offset, SEEK_SET); - int64_t ret = read(pFile->fd, buf, count); - lseek(pFile->fd, pos, SEEK_SET); + size_t pos = _lseek(pFile->fd, 0, SEEK_CUR); + _lseek(pFile->fd, offset, SEEK_SET); + int64_t ret = _read(pFile->fd, buf, count); + _lseek(pFile->fd, pos, SEEK_SET); #else int64_t ret = pread(pFile->fd, buf, count, offset); #endif @@ -428,7 +435,11 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { taosThreadRwlockRdlock(&(pFile->rwlock)); #endif assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + int64_t ret = _lseek(pFile->fd, offset, whence); +#else int64_t ret = lseek(pFile->fd, offset, whence); +#endif #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -567,12 +578,12 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in #ifdef WINDOWS - lseek(pFileIn->fd, (int32_t)(*offset), 0); + _lseek(pFileIn->fd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); + size_t rlen = _read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { @@ -586,7 +597,7 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in int64_t remain = size - writeLen; if (remain > 0) { - size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain); + size_t rlen = _read(pFileIn->fd, (void *)buffer, (size_t)remain); if (rlen <= 0) { return writeLen; } else { diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index e3791af618..24bc9d0b4c 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -37,6 +37,49 @@ typedef struct TdMemoryInfo { #ifdef WINDOWS #define tstrdup(str) _strdup(str) + +int32_t taosBackTrace(void **buffer, int32_t size) { + int32_t frame = 0; + return frame; +} + +#ifdef USE_ADDR2LINE +#include +#pragma comment(lib, "dbghelp.lib") + +void taosPrintBackTrace() { + #define MAX_STACK_FRAMES 20 + + void *pStack[MAX_STACK_FRAMES]; + + HANDLE process = GetCurrentProcess(); + SymInitialize(process, NULL, TRUE); + WORD frames = CaptureStackBackTrace(1, MAX_STACK_FRAMES, pStack, NULL); + + char buf_tmp[1024]; + for (WORD i = 0; i < frames; ++i) { + DWORD64 address = (DWORD64)(pStack[i]); + + DWORD64 displacementSym = 0; + char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)]; + PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer; + pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); + pSymbol->MaxNameLen = MAX_SYM_NAME; + + DWORD displacementLine = 0; + IMAGEHLP_LINE64 line; + //SymSetOptions(SYMOPT_LOAD_LINES); + line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); + + if (SymFromAddr(process, address, &displacementSym, pSymbol) && SymGetLineFromAddr64(process, address, &displacementLine, &line)) { + snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace %08" PRId64 " %s:%d %s\n", taosGetSelfPthreadId(), line.FileName, line.LineNumber, pSymbol->Name); + } else { + snprintf(buf_tmp,sizeof(buf_tmp),"BackTrace error: %d\n",GetLastError()); + } + write(1,buf_tmp,strlen(buf_tmp)); + } +} +#endif #else #define tstrdup(str) strdup(str) diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 7df4c26afd..d4cfe4fc39 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -68,9 +68,32 @@ int32_t tsem_wait(tsem_t* sem) { } int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { - int ret = 0; + struct timespec ts, rel; + FILETIME ft_before, ft_after; + int rc; - return ret; + rel.tv_sec = 0; + rel.tv_nsec = nanosecs; + + GetSystemTimeAsFileTime(&ft_before); + errno = 0; + rc = sem_timedwait(&sem, pthread_win32_getabstime_np(&ts, &rel)); + + /* This should have timed out */ + assert(errno == ETIMEDOUT); + assert(rc != 0); + GetSystemTimeAsFileTime(&ft_after); + // We specified a non-zero wait. Time must advance. + if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) + { + printf("nanoseconds: %d, rc: %d, errno: %d. before filetime: %d, %d; after filetime: %d, %d\n", + nanosecs, rc, errno, + (int)ft_before.dwLowDateTime, (int)ft_before.dwHighDateTime, + (int)ft_after.dwLowDateTime, (int)ft_after.dwHighDateTime); + printf("time must advance during sem_timedwait."); + return 1; + } + return 0; } #elif defined(_TD_DARWIN_64) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 105acb188a..572e2db6fd 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -718,7 +718,11 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { bzero((char *)&serverAdd, sizeof(serverAdd)); serverAdd.sin_family = AF_INET; +#ifdef WINDOWS + serverAdd.sin_addr.s_addr = INADDR_ANY; +#else serverAdd.sin_addr.s_addr = ip; +#endif serverAdd.sin_port = (uint16_t)htons(port); if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { @@ -882,6 +886,16 @@ void taosBlockSIGPIPE() { } uint32_t taosGetIpv4FromFqdn(const char *fqdn) { +#ifdef WINDOWS + // Initialize Winsock + WSADATA wsaData; + int iResult; + iResult = WSAStartup(MAKEWORD(2, 2), &wsaData); + if (iResult != 0) { + printf("WSAStartup failed: %d\n", iResult); + return 1; + } +#endif struct addrinfo hints = {0}; hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; @@ -899,12 +913,12 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { } else { #ifdef EAI_SYSTEM if (ret == EAI_SYSTEM) { - // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, strerror(errno)); + printf("failed to get the ip address, fqdn:%s, errno:%d, since:%s", fqdn, errno, strerror(errno)); } else { - // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret)); } #else - // printf("failed to get the ip address, fqdn:%s, since:%s", fqdn, gai_strerror(ret)); + printf("failed to get the ip address, fqdn:%s, ret:%d, since:%s", fqdn, ret, gai_strerror(ret)); #endif return 0xFFFFFFFF; } diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index fd898984ed..185daf9e45 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -29,10 +29,10 @@ struct SEncoderNode { }; struct SDecoderNode { - SDecoderNode* pNext; - const uint8_t* data; - uint32_t size; - uint32_t pos; + SDecoderNode* pNext; + uint8_t* data; + uint32_t size; + uint32_t pos; }; void tEncoderInit(SEncoder* pEncoder, uint8_t* data, uint32_t size) { @@ -52,7 +52,7 @@ void tEncoderClear(SEncoder* pCoder) { memset(pCoder, 0, sizeof(*pCoder)); } -void tDecoderInit(SDecoder* pDecoder, const uint8_t* data, uint32_t size) { +void tDecoderInit(SDecoder* pDecoder, uint8_t* data, uint32_t size) { pDecoder->data = data; pDecoder->size = size; pDecoder->pos = 0; diff --git a/tools/taos-tools b/tools/taos-tools index 772aef458f..2c4a1c8332 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 772aef458fec13804c654e59c7a248b55ba3807b +Subproject commit 2c4a1c83322b983881aea93ec2b51e7df826125a