From 9ba1a32dc8b35e3ce6259f282e6dcf693fe10089 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Oct 2023 14:14:44 +0800 Subject: [PATCH 01/21] change NumOfTaskQueueThreads size --- source/common/src/tglobal.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0a155f4ea1..fc95e9335c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -486,9 +486,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); - if (tsNumOfTaskQueueThreads >= 10) { - tsNumOfTaskQueueThreads = 10; - } if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; return 0; From 13b6d3174e06125e1d8580cab0ea9c6257b2aa61 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Nov 2023 10:35:25 +0800 Subject: [PATCH 02/21] change NumOfTaskQueueThreads size --- source/common/src/tglobal.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fc95e9335c..4c3fb065b4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -486,6 +486,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); + + if (tsNumOfTaskQueueThreads >= 50) { + tsNumOfTaskQueueThreads = 50; + } if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; return 0; @@ -1798,4 +1802,4 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { uInfo("all debug flag are set to %d", flag); } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } +int8_t taosGranted() { return atomic_load_8(&tsGrant); } \ No newline at end of file From 4c36d1ee113a8fc254e3607f9ffd969b55a5401f Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 17 Nov 2023 14:27:56 +0800 Subject: [PATCH 03/21] fix: error msg when invalid operator --- source/libs/parser/src/parTranslater.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1a65a29259..3e77feb0c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1439,8 +1439,10 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - if (TSDB_CODE_SUCCESS != scalarGetOperatorResultType(pOp)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pOp->node.aliasName); + int32_t res = scalarGetOperatorResultType(pOp); + if (TSDB_CODE_SUCCESS != res) { + pCxt->errCode = res; + return DEAL_RES_CONTINUE; } return DEAL_RES_CONTINUE; From 9916e0d24cb5dff6baad304361e9b8432ef10f57 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 20 Nov 2023 09:55:59 +0800 Subject: [PATCH 04/21] build release version on windows --- cmake/cmake.define | 10 +++++++++- contrib/CMakeLists.txt | 32 +++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 44b36d0efa..7710c071eb 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -97,7 +97,15 @@ ENDIF() SET(JEMALLOC_ENABLED OFF) IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd") + IF (${CMAKE_BUILD_TYPE} MATCHES "Release") + MESSAGE("${Green} will build Release version! ${ColourReset}") + SET(COMMON_FLAGS "/W3 /D_WIN32 /DWIN32 /Zi- /O2 /GL /MD") + + ELSE () + MESSAGE("${Green} will build Debug version! ${ColourReset}") + SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd") + ENDIF() + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c5715bd53f..1b0a091e9d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -317,7 +317,8 @@ if (${BUILD_WITH_ROCKSDB}) SET(CMAKE_BUILD_TYPE Release) endif() endif(${TD_LINUX}) - MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS}) if(${TD_DARWIN}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") @@ -329,8 +330,12 @@ if (${BUILD_WITH_ROCKSDB}) if (${TD_WINDOWS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819") option(WITH_JNI "" OFF) - option(WITH_MD_LIBRARY "build with MD" OFF) + if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd") + message("Rocksdb build runtime lib use /MT or /MTd") + option(WITH_MD_LIBRARY "build with MD" OFF) + endif() set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) endif(${TD_WINDOWS}) @@ -361,9 +366,11 @@ if (${BUILD_WITH_ROCKSDB}) ) else() if (NOT ${TD_LINUX}) - MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS}) if(${TD_DARWIN}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=maybe-uninitialized") endif(${TD_DARWIN}) if (${TD_DARWIN_ARM64}) @@ -372,8 +379,12 @@ if (${BUILD_WITH_ROCKSDB}) if (${TD_WINDOWS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819") option(WITH_JNI "" OFF) - option(WITH_MD_LIBRARY "build with MD" OFF) + if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd") + message("Rocksdb build runtime lib use /MT or /MTd") + option(WITH_MD_LIBRARY "build with MD" OFF) + endif() set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) endif(${TD_WINDOWS}) @@ -456,7 +467,9 @@ endif(${BUILD_WITH_NURAFT}) # pthread if(${BUILD_PTHREAD}) - set(CMAKE_BUILD_TYPE debug) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE Release) + endif() add_definitions(-DPTW32_STATIC_LIB) add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) @@ -640,13 +653,18 @@ if(${BUILD_GEOS}) if(${TD_LINUX}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}") - IF ("${CMAKE_BUILD_TYPE}" STREQUAL "") + if ("${CMAKE_BUILD_TYPE}" STREQUAL "") SET(CMAKE_BUILD_TYPE Release) endif() endif(${TD_LINUX}) option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF) add_subdirectory(geos EXCLUDE_FROM_ALL) - unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + else () + unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD + endif(${TD_WINDOWS}) target_include_directories( geos_c PUBLIC $ From 7579addf82b2ac127c08499db240c59866fa3dec Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 21 Nov 2023 01:43:17 +0800 Subject: [PATCH 05/21] fix case: invalid operate --- tests/system-test/0-others/view/non_marterial_view/test_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/view/non_marterial_view/test_view.py b/tests/system-test/0-others/view/non_marterial_view/test_view.py index afb2476305..4b829b4049 100644 --- a/tests/system-test/0-others/view/non_marterial_view/test_view.py +++ b/tests/system-test/0-others/view/non_marterial_view/test_view.py @@ -162,7 +162,7 @@ class TDTestCase: assert('TIMESTAMP' in data_type_list and 'INT' in data_type_list and 'INT UNSIGNED' in data_type_list and 'BIGINT' in data_type_list and 'BIGINT UNSIGNED' in data_type_list and 'FLOAT' in data_type_list and 'DOUBLE' in data_type_list and 'VARCHAR' in data_type_list and 'SMALLINT' in data_type_list and 'SMALLINT UNSIGNED' in data_type_list and 'TINYINT' in data_type_list and 'TINYINT UNSIGNED' in data_type_list and 'BOOL' in data_type_list and 'VARCHAR' in data_type_list and 'NCHAR' in data_type_list and 'GEOMETRY' in data_type_list and 'VARBINARY' in data_type_list) tdSql.execute("create view v2 as select * from tb where c1 >5 and c7 like '%ab%';") self.check_view_num(2) - tdSql.error("create view v3 as select * from tb where c1 like '%ab%';", expectErrInfo='Invalid value type') + tdSql.error("create view v3 as select * from tb where c1 like '%ab%';", expectErrInfo='Invalid operation') tdSql.execute("create view v3 as select first(ts), sum(c1) from tb group by c2 having avg(c4) > 0;") tdSql.execute("create view v4 as select _wstart,sum(c6) from tb interval(10s);") tdSql.execute("create view v5 as select * from tb join v2 on tb.ts = v2.ts;") From b3ca673fe93a93aa2eddb49f380f9909667a9aca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 21 Nov 2023 14:35:41 +0800 Subject: [PATCH 06/21] fix:test case for windows --- source/dnode/snode/src/snode.c | 1 + utils/test/c/tmq_taosx_ci.c | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 6f5b370826..748888193b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -137,6 +137,7 @@ FAIL: } void sndClose(SSnode *pSnode) { + stopRsync(); streamMetaNotifyClose(pSnode->pMeta); streamMetaCommit(pSnode->pMeta); streamMetaClose(pSnode->pMeta); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index ff89bb1f75..7dbde60632 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -30,7 +30,7 @@ typedef struct { int meta; int srcVgroups; int dstVgroups; - char dir[64]; + char dir[256]; } Config; Config g_conf = {0}; @@ -409,6 +409,30 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); +#ifdef WINDOWS + pRes = taos_query(pConn, + "CREATE STABLE `meters_summary` (`_wstart` TIMESTAMP, `current` FLOAT, `groupid` INT, `location` VARCHAR(16)) TAGS (`group_id` BIGINT UNSIGNED)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, + " CREATE TABLE `t_d2a450ee819dcf7576f0282d9ac22dbc` USING `meters_summary` (`group_id`) TAGS (13135550082773579308)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into t_d2a450ee819dcf7576f0282d9ac22dbc values (now, 120, 1, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); +#else pRes = taos_query(pConn, "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 into meters_summary as select _wstart, max(current) as current, " "groupid, location from meters partition by groupid, location interval(10m)"); @@ -417,6 +441,7 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { return -1; } taos_free_result(pRes); +#endif pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); if (taos_errno(pRes) != 0) { From a92602b667e737a59dd586df0c670c031fbddfa3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 21 Nov 2023 15:23:28 +0800 Subject: [PATCH 07/21] fix mem leak --- include/libs/stream/tstream.h | 119 +++++++++++---------- source/dnode/vnode/src/tq/tq.c | 119 ++++++++++----------- source/libs/stream/inc/streamInt.h | 44 ++++---- source/libs/stream/src/streamCheckpoint.c | 91 ++++++++-------- source/libs/stream/src/streamDispatch.c | 121 +++++++++++++--------- source/libs/stream/src/streamTask.c | 43 ++++---- 6 files changed, 288 insertions(+), 249 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6e191e412d..c70a333099 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -34,17 +34,16 @@ extern "C" { #define SIZE_IN_MiB(_v) ((_v) / ONE_MiB_F) #define SIZE_IN_KiB(_v) ((_v) / ONE_KiB_F) +#define TASK_DOWNSTREAM_READY 0x0 +#define TASK_DOWNSTREAM_NOT_READY 0x1 +#define TASK_DOWNSTREAM_NOT_LEADER 0x2 +#define TASK_UPSTREAM_NEW_STAGE 0x3 -#define TASK_DOWNSTREAM_READY 0x0 -#define TASK_DOWNSTREAM_NOT_READY 0x1 -#define TASK_DOWNSTREAM_NOT_LEADER 0x2 -#define TASK_UPSTREAM_NEW_STAGE 0x3 +#define NODE_ROLE_UNINIT 0x1 +#define NODE_ROLE_LEADER 0x2 +#define NODE_ROLE_FOLLOWER 0x3 -#define NODE_ROLE_UNINIT 0x1 -#define NODE_ROLE_LEADER 0x2 -#define NODE_ROLE_FOLLOWER 0x3 - -#define HAS_RELATED_FILLHISTORY_TASK(_t) ((_t)->hTaskInfo.id.taskId != 0) +#define HAS_RELATED_FILLHISTORY_TASK(_t) ((_t)->hTaskInfo.id.taskId != 0) #define CLEAR_RELATED_FILLHISTORY_TASK(_t) \ do { \ (_t)->hTaskInfo.id.taskId = 0; \ @@ -159,7 +158,7 @@ typedef struct { typedef struct { int8_t type; int64_t ver; - SArray* submits; // SArray + SArray* submits; // SArray } SStreamMergedSubmit; typedef struct { @@ -251,7 +250,7 @@ typedef struct { } SScanhistoryDataInfo; typedef struct { - int32_t idleDuration; // idle time before use time slice the continue execute scan-history + int32_t idleDuration; // idle time before use time slice the continue execute scan-history int32_t numOfTicks; tmr_h pTimer; int32_t execCount; @@ -301,10 +300,10 @@ typedef struct SStreamTaskId { typedef struct SCheckpointInfo { int64_t startTs; int64_t checkpointId; - int64_t checkpointVer; // latest checkpointId version - int64_t processedVer; // already processed ver, that has generated results version. - int64_t nextProcessVer; // current offset in WAL, not serialize it - int64_t failedId; // record the latest failed checkpoint id + int64_t checkpointVer; // latest checkpointId version + int64_t processedVer; // already processed ver, that has generated results version. + int64_t nextProcessVer; // current offset in WAL, not serialize it + int64_t failedId; // record the latest failed checkpoint id } SCheckpointInfo; typedef struct SStreamStatus { @@ -331,22 +330,22 @@ typedef struct SSTaskBasicInfo { int32_t selfChildId; int32_t totalLevel; int8_t taskLevel; - int8_t fillHistory; // is fill history task or not - int64_t triggerParam; // in msec + int8_t fillHistory; // is fill history task or not + int64_t triggerParam; // in msec } SSTaskBasicInfo; typedef struct SStreamDispatchReq SStreamDispatchReq; -typedef struct STokenBucket STokenBucket; -typedef struct SMetaHbInfo SMetaHbInfo; +typedef struct STokenBucket STokenBucket; +typedef struct SMetaHbInfo SMetaHbInfo; typedef struct SDispatchMsgInfo { - SStreamDispatchReq* pData; // current dispatch data - int8_t dispatchMsgType; - int16_t msgType; // dispatch msg type - int32_t retryCount; // retry send data count - int64_t startTs; // dispatch start time, record total elapsed time for dispatch - SArray* pRetryList; // current dispatch successfully completed node of downstream - void* pTimer; // used to dispatch data after a given time duration + SStreamDispatchReq* pData; // current dispatch data + int8_t dispatchMsgType; + int16_t msgType; // dispatch msg type + int32_t retryCount; // retry send data count + int64_t startTs; // dispatch start time, record total elapsed time for dispatch + SArray* pRetryList; // current dispatch successfully completed node of downstream + void* pTimer; // used to dispatch data after a given time duration } SDispatchMsgInfo; typedef struct STaskQueue { @@ -355,8 +354,8 @@ typedef struct STaskQueue { } STaskQueue; typedef struct STaskSchedInfo { - int8_t status; - void* pTimer; + int8_t status; + void* pTimer; } STaskSchedInfo; typedef struct SSinkRecorder { @@ -390,7 +389,7 @@ typedef struct SHistoryTaskInfo { int32_t tickCount; int32_t retryTimes; int32_t waitInterval; - int64_t haltVer; // offset in wal when halt the stream task + int64_t haltVer; // offset in wal when halt the stream task } SHistoryTaskInfo; typedef struct STaskOutputInfo { @@ -456,9 +455,9 @@ typedef struct STaskStartInfo { int64_t startTs; int64_t readyTs; int32_t tasksWillRestart; - int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. - SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing - SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed + int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. + SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing + SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed int64_t elapsedTime; } STaskStartInfo; @@ -531,7 +530,7 @@ struct SStreamDispatchReq { int64_t stage; // nodeId from upstream task int64_t streamId; int32_t taskId; - int32_t msgId; // msg id to identify if the incoming msg from the same sender + int32_t msgId; // msg id to identify if the incoming msg from the same sender int32_t srcVgId; int32_t upstreamTaskId; int32_t upstreamChildId; @@ -657,22 +656,22 @@ typedef struct STaskStatusEntry { int32_t status; int32_t stage; int32_t nodeId; - int64_t verStart; // start version in WAL, only valid for source task - int64_t verEnd; // end version in WAL, only valid for source task - int64_t processedVer; // only valid for source task - int64_t activeCheckpointId; // current active checkpoint id - bool checkpointFailed; // denote if the checkpoint is failed or not - double inputQUsed; // in MiB + int64_t verStart; // start version in WAL, only valid for source task + int64_t verEnd; // end version in WAL, only valid for source task + int64_t processedVer; // only valid for source task + int64_t activeCheckpointId; // current active checkpoint id + bool checkpointFailed; // denote if the checkpoint is failed or not + double inputQUsed; // in MiB double inputRate; - double sinkQuota; // existed quota size for sink task - double sinkDataSize; // sink to dest data size + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dest data size } STaskStatusEntry; typedef struct SStreamHbMsg { int32_t vgId; int32_t numOfTasks; - SArray* pTaskStatus; // SArray - SArray* pUpdateNodes; // SArray, needs update the epsets in stream tasks for those nodes. + SArray* pTaskStatus; // SArray + SArray* pUpdateNodes; // SArray, needs update the epsets in stream tasks for those nodes. } SStreamHbMsg; int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp); @@ -696,7 +695,7 @@ typedef struct SNodeUpdateInfo { } SNodeUpdateInfo; typedef struct SStreamTaskNodeUpdateMsg { - int32_t transId; // to identify the msg + int32_t transId; // to identify the msg int64_t streamId; int32_t taskId; SArray* pNodeList; // SArray @@ -753,10 +752,10 @@ const char* streamTaskGetStatusStr(ETaskStatus status); void streamTaskResetStatus(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask); -void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); +void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); // recover and fill history -void streamTaskCheckDownstream(SStreamTask* pTask); +void streamTaskCheckDownstream(SStreamTask* pTask); int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); @@ -784,18 +783,18 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer) int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); // common -int32_t streamRestoreParam(SStreamTask* pTask); -void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); -void streamTaskResume(SStreamTask* pTask); -void streamTaskEnablePause(SStreamTask* pTask); -int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); -void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); -void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); -void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask); -int32_t streamTaskReleaseState(SStreamTask* pTask); -int32_t streamTaskReloadState(SStreamTask* pTask); -void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); -void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); +int32_t streamRestoreParam(SStreamTask* pTask); +void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); +void streamTaskResume(SStreamTask* pTask); +void streamTaskEnablePause(SStreamTask* pTask); +int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); +void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); +void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); +void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask); +int32_t streamTaskReleaseState(SStreamTask* pTask); +int32_t streamTaskReloadState(SStreamTask* pTask); +void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); +void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask); void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); @@ -804,7 +803,7 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st); -int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); +int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); // agg level int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, SRpcHandleInfo* pInfo); @@ -842,7 +841,7 @@ void streamMetaResetStartInfo(STaskStartInfo* pMeta); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); -void streamTaskClearCheckInfo(SStreamTask* pTask); +void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg); int32_t streamAlignTransferState(SStreamTask* pTask); int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId); int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6dbeaef6cb..5671131741 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -88,7 +88,7 @@ void tqDestroyTqHandle(void* data) { taosMemoryFree(pData->msg); pData->msg = NULL; } - if (pData->block != NULL){ + if (pData->block != NULL) { blockDataDestroy(pData->block); } } @@ -591,9 +591,9 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosWLockLatch(&pTq->lock); bool exec = tqIsHandleExec(pHandle); - if(exec){ + if (exec) { tqInfo("vgId:%d, topic:%s, subscription is executing, delete wait for 10ms and retry, pHandle:%p", vgId, - pHandle->subKey, pHandle); + pHandle->subKey, pHandle); taosWUnLockLatch(&pTq->lock); taosMsleep(10); continue; @@ -710,12 +710,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, &handle); taosWUnLockLatch(&pTq->lock); } else { - while(1){ + while (1) { taosWLockLatch(&pTq->lock); bool exec = tqIsHandleExec(pHandle); - if(exec){ - tqInfo("vgId:%d, topic:%s, subscription is executing, sub wait for 10ms and retry, pHandle:%p", pTq->pVnode->config.vgId, - pHandle->subKey, pHandle); + if (exec) { + tqInfo("vgId:%d, topic:%s, subscription is executing, sub wait for 10ms and retry, pHandle:%p", + pTq->pVnode->config.vgId, pHandle->subKey, pHandle); taosWUnLockLatch(&pTq->lock); taosMsleep(10); continue; @@ -724,7 +724,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d no switch consumer:0x%" PRIx64 " remains, because redo wal log", req.vgId, req.newConsumerId); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, - req.newConsumerId); + req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_store_32(&pHandle->epoch, 0); tqUnregisterPushHandle(pTq, pHandle); @@ -856,11 +856,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); } -// // reset the task status from unfinished transaction -// if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { -// tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr); -// pTask->status.taskStatus = TASK_STATUS__READY; -// } + // // reset the task status from unfinished transaction + // if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { + // tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr); + // pTask->status.taskStatus = TASK_STATUS__READY; + // } streamTaskResetUpstreamStageInfo(pTask); streamSetupScheduleTrigger(pTask); @@ -896,9 +896,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { } int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { - char* msgStr = pMsg->pCont; - char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); SStreamMeta* pMeta = pTq->pStreamMeta; SStreamTaskCheckReq req; @@ -922,8 +922,9 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { // only the leader node handle the check request if (pMeta->role == NODE_ROLE_FOLLOWER) { - tqError("s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", - taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); + tqError( + "s-task:0x%x invalid check msg from upstream:0x%x(vgId:%d), vgId:%d is follower, not handle check status msg", + taskId, req.upstreamTaskId, req.upstreamNodeId, pMeta->vgId); rsp.status = TASK_DOWNSTREAM_NOT_LEADER; } else { SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); @@ -933,7 +934,8 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { char* p = NULL; streamTaskGetStatus(pTask, &p); - tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", + tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 + ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = TASK_DOWNSTREAM_NOT_READY; @@ -999,7 +1001,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms tqDebug("vgId:%d receive new stream task deploy msg, start to build stream task", vgId); // 1.deserialize msg and build task - int32_t size = sizeof(SStreamTask); + int32_t size = sizeof(SStreamTask); SStreamTask* pTask = taosMemoryCalloc(1, size); if (pTask == NULL) { tqError("vgId:%d failed to create stream task due to out of memory, alloc size:%d", vgId, size); @@ -1029,7 +1031,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms streamMetaWUnLock(pStreamMeta); if (code < 0) { - tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, tstrerror(code)); + tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks, + tstrerror(code)); tFreeStreamTask(pTask); return code; } @@ -1069,7 +1072,7 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer; // if it's an source task, extract the last version in wal. - SVersionRange *pRange = &pTask->dataRange.range; + SVersionRange* pRange = &pTask->dataRange.range; bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer); pTask->execInfo.step2Start = taosGetTimestampMs(); @@ -1081,7 +1084,7 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask } else { STimeWindow* pWindow = &pTask->dataRange.window; tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " window:%" PRId64 "-%" PRId64 - ", do secondary scan-history from WAL after halt the related stream task:%s", + ", do secondary scan-history from WAL after halt the related stream task:%s", id, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pWindow->skey, pWindow->ekey, pStreamTask->id.idStr); ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING); @@ -1095,7 +1098,7 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE); - /*int8_t status = */streamTaskSetSchedStatusInactive(pTask); + /*int8_t status = */ streamTaskSetSchedStatusInactive(pTask); // now the fill-history task starts to scan data from wal files. int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); @@ -1124,7 +1127,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { streamTaskGetStatus(pTask, &pStatus); // avoid multi-thread exec - while(1) { + while (1) { int32_t sentinel = atomic_val_compare_exchange_32(&pTask->status.inScanHistorySentinel, 0, 1); if (sentinel != 0) { tqDebug("s-task:%s already in scan-history func, wait for 100ms, and try again", id); @@ -1171,7 +1174,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { return 0; } - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampMs(); SScanhistoryDataInfo retInfo = streamScanHistoryData(pTask, st); double el = (taosGetTimestampMs() - st) / 1000.0; @@ -1204,13 +1207,13 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El); if (pTask->info.fillHistory) { - SStreamTask* pStreamTask = NULL; + SStreamTask* pStreamTask = NULL; // 1. get the related stream task pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); if (pStreamTask == NULL) { - tqError("failed to find s-task:0x%"PRIx64", it may have been destroyed, drop related fill-history task:%s", - pTask->streamTaskId.taskId, pTask->id.idStr); + tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", + pTask->streamTaskId.taskId, pTask->id.idStr); tqDebug("s-task:%s fill-history task set status to be dropping", id); streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id); @@ -1327,7 +1330,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId); - if (pTask != NULL) { // even in halt status, the data in inputQ must be processed + if (pTask != NULL) { // even in halt status, the data in inputQ must be processed char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, @@ -1484,7 +1487,6 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SINK) { if (status == TASK_STATUS__UNINIT) { - } streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; @@ -1526,12 +1528,12 @@ int32_t tqProcessTaskResumeReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, pReq->taskId); - int32_t code = tqProcessTaskResumeImpl(pTq, pTask, sversion, pReq->igUntreated); + int32_t code = tqProcessTaskResumeImpl(pTq, pTask, sversion, pReq->igUntreated); if (code != 0) { return code; } - STaskId* pHTaskId = &pTask->hTaskInfo.id; + STaskId* pHTaskId = &pTask->hTaskInfo.id; SStreamTask* pHistoryTask = streamMetaAcquireTask(pTq->pStreamMeta, pHTaskId->streamId, pHTaskId->taskId); if (pHistoryTask) { code = tqProcessTaskResumeImpl(pTq, pHistoryTask, sversion, pReq->igUntreated); @@ -1551,7 +1553,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { tDecodeStreamRetrieveReq(&decoder, &req); tDecoderClear(&decoder); - int32_t vgId = pTq->pStreamMeta->vgId; + int32_t vgId = pTq->pStreamMeta->vgId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, req.dstTaskId); if (pTask == NULL) { tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", vgId, @@ -1596,10 +1598,10 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { tqDebug("vgId:%d receive dispatch msg to s-task:0x%" PRIx64 "-0x%x", vgId, req.streamId, taskId); // for test purpose -// if (req.type == STREAM_INPUT__CHECKPOINT_TRIGGER) { -// code = TSDB_CODE_STREAM_TASK_NOT_EXIST; -// goto FAIL; -// } + // if (req.type == STREAM_INPUT__CHECKPOINT_TRIGGER) { + // code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + // goto FAIL; + // } SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.streamId, taskId); if (pTask != NULL) { @@ -1669,7 +1671,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) tqDebug("vgId:%d not leader, ignore checkpoint-source msg, s-task:0x%x", vgId, req.taskId); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } @@ -1677,7 +1679,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) tqDebug("vgId:%d checkpoint-source msg received during restoring, s-task:0x%x ignore it", vgId, req.taskId); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } @@ -1689,7 +1691,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) tqError("vgId:%d failed to decode checkpoint-source msg, code:%s", vgId, tstrerror(code)); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return code; } tDecoderClear(&decoder); @@ -1700,22 +1702,23 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) req.taskId); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. if (pTask->status.downstreamReady != 1) { - pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id + pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id pTask->checkpointingId = req.checkpointId; qError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 - ", set it failure", pTask->id.idStr, req.checkpointId); + ", set it failure", + pTask->id.idStr, req.checkpointId); streamMetaReleaseTask(pMeta, pTask); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } @@ -1725,14 +1728,14 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) { tqError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure", - pTask->id.idStr, req.checkpointId); + pTask->id.idStr, req.checkpointId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return TSDB_CODE_SUCCESS; } @@ -1766,13 +1769,13 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamMetaWUnLock(pMeta); qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", total checkpoint reqs:%d", - pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, total); + pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, total); code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask, 1); if (code != TSDB_CODE_SUCCESS) { SRpcMsg rsp = {0}; buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); - tmsgSendRsp(&rsp); // error occurs + tmsgSendRsp(&rsp); // error occurs return code; } @@ -1837,7 +1840,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { streamMetaWLock(pMeta); // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. - STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; + STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL || *ppTask == NULL) { tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, @@ -1861,7 +1864,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { } STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; - void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); + void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); if (exist != NULL) { tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, req.transId); @@ -1931,7 +1934,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { streamMetaWUnLock(pMeta); } else { if (!pTq->pVnode->restored) { - tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); + tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", + vgId); pMeta->startInfo.tasksWillRestart = 0; streamMetaWUnLock(pMeta); } else { @@ -1947,7 +1951,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { // the following procedure consume many CPU resource, result in the re-election of leader // with high probability. So we employ it as a test case for the stream processing framework, with // checkpoint/restart/nodeUpdate etc. - while(1) { + while (1) { int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); if (startVal == 0) { break; @@ -1998,13 +2002,13 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*) pMsg->pCont; + SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg->pCont; SStreamMeta* pMeta = pTq->pStreamMeta; SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); if (pTask == NULL) { - tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, - pReq->taskId); + tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, pReq->taskId); return TSDB_CODE_SUCCESS; } @@ -2012,8 +2016,7 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { // clear flag set during do checkpoint, and open inputQ for all upstream tasks if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { - streamTaskClearCheckInfo(pTask); - taosArrayClear(pTask->pReadyMsgList); + streamTaskClearCheckInfo(pTask, true); streamTaskSetStatusReady(pTask); } diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index b76a967d0d..b9e9db8662 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -18,9 +18,9 @@ #include "executor.h" #include "query.h" -#include "tstream.h" #include "streamBackendRocksdb.h" #include "trpc.h" +#include "tstream.h" #ifdef __cplusplus extern "C" { @@ -32,13 +32,13 @@ extern "C" { #define MAX_RETRY_LAUNCH_HISTORY_TASK 40 #define RETRY_LAUNCH_INTERVAL_INC_RATE 1.2 -#define MAX_BLOCK_NAME_NUM 1024 -#define DISPATCH_RETRY_INTERVAL_MS 300 -#define MAX_CONTINUE_RETRY_COUNT 5 +#define MAX_BLOCK_NAME_NUM 1024 +#define DISPATCH_RETRY_INTERVAL_MS 300 +#define MAX_CONTINUE_RETRY_COUNT 5 -#define META_HB_CHECK_INTERVAL 200 -#define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec -#define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1) +#define META_HB_CHECK_INTERVAL 200 +#define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec +#define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1) #define STREAM_TASK_QUEUE_CAPACITY 20480 #define STREAM_TASK_QUEUE_CAPACITY_IN_SIZE (30) @@ -86,13 +86,13 @@ struct SStreamQueue { }; extern SStreamGlobalEnv streamEnv; -extern int32_t streamBackendId; -extern int32_t streamBackendCfWrapperId; +extern int32_t streamBackendId; +extern int32_t streamBackendCfWrapperId; -void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); -int32_t streamDispatchStreamBlock(SStreamTask* pTask); -void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); -int32_t getNumOfDispatchBranch(SStreamTask* pTask); +void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); +int32_t streamDispatchStreamBlock(SStreamTask* pTask); +void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); +int32_t getNumOfDispatchBranch(SStreamTask* pTask); int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); @@ -113,19 +113,23 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); -int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, int32_t* blockSize); +int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, + int32_t* blockSize); int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem); void streamQueueItemIncSize(const SStreamQueueItem* pItem, int32_t size); const char* streamQueueItemGetTypeStr(int32_t type); SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); -int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, int32_t* pLen); +int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, + int32_t* pLen); int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, SStreamScanHistoryFinishReq* pReq); int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTaskFillHistoryFinished(SStreamTask* pTask); int32_t streamTransferStateToStreamTask(SStreamTask* pTask); +void streamClearChkptReadyMsg(SStreamTask* pTask); + int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo); @@ -139,17 +143,17 @@ void* streamQueueNextItem(SStreamQueue* pQueue); void streamFreeQitem(SStreamQueueItem* data); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); -typedef enum UPLOAD_TYPE{ +typedef enum UPLOAD_TYPE { UPLOAD_DISABLE = -1, UPLOAD_S3 = 0, UPLOAD_RSYNC = 1, } UPLOAD_TYPE; UPLOAD_TYPE getUploadType(); -int uploadCheckpoint(char* id, char* path); -int downloadCheckpoint(char* id, char* path); -int deleteCheckpoint(char* id); -int deleteCheckpointFile(char* id, char* name); +int uploadCheckpoint(char* id, char* path); +int downloadCheckpoint(char* id, char* path); +int deleteCheckpoint(char* id); +int deleteCheckpointFile(char* id, char* name); int32_t onNormalTaskReady(SStreamTask* pTask); int32_t onScanhistoryTaskReady(SStreamTask* pTask); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 6201329b95..c392b17c74 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "streamInt.h" -#include "rsync.h" #include "cos.h" +#include "rsync.h" +#include "streamInt.h" int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; @@ -122,7 +122,7 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint pBlock->info.rows = 1; pBlock->info.childId = pTask->info.selfChildId; - pChkpoint->blocks = taosArrayInit(4, sizeof(SSDataBlock));//pBlock; + pChkpoint->blocks = taosArrayInit(4, sizeof(SSDataBlock)); // pBlock; taosArrayPush(pChkpoint->blocks, pBlock); taosMemoryFree(pBlock); @@ -169,10 +169,10 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); - int64_t checkpointId = pDataBlock->info.version; + int64_t checkpointId = pDataBlock->info.version; const char* id = pTask->id.idStr; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; // set task status if (streamTaskGetStatus(pTask, NULL) != TASK_STATUS__CK) { @@ -184,7 +184,7 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc } } - { // todo: remove this when the pipeline checkpoint generating is used. + { // todo: remove this when the pipeline checkpoint generating is used. SStreamMeta* pMeta = pTask->pMeta; streamMetaWLock(pMeta); @@ -195,10 +195,11 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc streamMetaWUnLock(pMeta); } - //todo fix race condition: set the status and append checkpoint block + // todo fix race condition: set the status and append checkpoint block int32_t taskLevel = pTask->info.taskLevel; if (taskLevel == TASK_LEVEL__SOURCE) { - if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || + pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); continueDispatchCheckpointBlock(pBlock, pTask); } else { // only one task exists, no need to dispatch downstream info @@ -221,20 +222,21 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); if (notReady > 0) { stDebug("s-task:%s received checkpoint block, idx:%d, %d upstream tasks not send checkpoint info yet, total:%d", - id, pTask->info.selfChildId, notReady, num); + id, pTask->info.selfChildId, notReady, num); streamFreeQitem((SStreamQueueItem*)pBlock); return code; } if (taskLevel == TASK_LEVEL__SINK) { stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, send ready msg to upstream", - id, num); + id, num); streamFreeQitem((SStreamQueueItem*)pBlock); streamTaskBuildCheckpoint(pTask); } else { stDebug( "s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, dispatch checkpoint msg " - "downstream", id, num); + "downstream", + id, num); // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task // can start local checkpoint procedure @@ -262,7 +264,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { if (notReady == 0) { stDebug("s-task:%s all downstream tasks have completed the checkpoint, start to do checkpoint for current task", - pTask->id.idStr); + pTask->id.idStr); appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT); } else { int32_t total = streamTaskGetNumOfDownstream(pTask); @@ -272,13 +274,17 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { return 0; } -void streamTaskClearCheckInfo(SStreamTask* pTask) { - pTask->checkpointingId = 0; // clear the checkpoint id +void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { + pTask->checkpointingId = 0; // clear the checkpoint id pTask->chkInfo.failedId = 0; pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->checkpointNotReadyTasks = 0; pTask->checkpointAlignCnt = 0; - streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks + streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks + + if (clearChkpReadyMsg) { + streamClearChkptReadyMsg(pTask); + } } int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { @@ -288,7 +294,7 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { streamMetaWLock(pMeta); for (int32_t i = 0; i < taosArrayGetSize(pMeta->pTaskList); ++i) { - STaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId* pId = taosArrayGet(pMeta->pTaskList, i); SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); if (ppTask == NULL) { continue; @@ -305,7 +311,7 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { p->chkInfo.checkpointId = p->checkpointingId; p->chkInfo.checkpointVer = p->chkInfo.processedVer; - streamTaskClearCheckInfo(p); + streamTaskClearCheckInfo(p, false); char* str = NULL; streamTaskGetStatus(p, &str); @@ -315,7 +321,7 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId) { stDebug("s-task:%s vgId:%d save task status failed, since handle event failed", p->id.idStr, vgId); streamMetaWUnLock(pMeta); return -1; - } else { // save the task + } else { // save the task streamMetaSaveTask(pMeta, p); } @@ -372,33 +378,32 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // todo: let's retry send rsp to upstream/mnode stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, - pTask->checkpointingId, tstrerror(code)); + pTask->checkpointingId, tstrerror(code)); } return code; } -static int uploadCheckpointToS3(char* id, char* path){ +static int uploadCheckpointToS3(char* id, char* path) { TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) return -1; TdDirEntryPtr de = NULL; while ((de = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || - taosDirEntryIsDir(de)) continue; + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || taosDirEntryIsDir(de)) continue; char filename[PATH_MAX] = {0}; - if(path[strlen(path) - 1] == TD_DIRSEP_CHAR){ + if (path[strlen(path) - 1] == TD_DIRSEP_CHAR) { snprintf(filename, sizeof(filename), "%s%s", path, name); - }else{ + } else { snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name); } char object[PATH_MAX] = {0}; snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); - if(s3PutObjectFromFile2(filename, object) != 0){ + if (s3PutObjectFromFile2(filename, object) != 0) { taosCloseDir(&pDir); return -1; } @@ -409,59 +414,59 @@ static int uploadCheckpointToS3(char* id, char* path){ return 0; } -UPLOAD_TYPE getUploadType(){ - if(strlen(tsSnodeAddress) != 0){ +UPLOAD_TYPE getUploadType() { + if (strlen(tsSnodeAddress) != 0) { return UPLOAD_RSYNC; - }else if(tsS3StreamEnabled){ + } else if (tsS3StreamEnabled) { return UPLOAD_S3; - }else{ + } else { return UPLOAD_DISABLE; } } -int uploadCheckpoint(char* id, char* path){ - if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ +int uploadCheckpoint(char* id, char* path) { + if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { stError("uploadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeAddress) != 0){ + if (strlen(tsSnodeAddress) != 0) { return uploadRsync(id, path); - }else if(tsS3StreamEnabled){ + } else if (tsS3StreamEnabled) { return uploadCheckpointToS3(id, path); } return 0; } -int downloadCheckpoint(char* id, char* path){ - if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ +int downloadCheckpoint(char* id, char* path) { + if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { stError("downloadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeAddress) != 0){ + if (strlen(tsSnodeAddress) != 0) { return downloadRsync(id, path); - }else if(tsS3StreamEnabled){ + } else if (tsS3StreamEnabled) { return s3GetObjectsByPrefix(id, path); } return 0; } -int deleteCheckpoint(char* id){ - if(id == NULL || strlen(id) == 0){ +int deleteCheckpoint(char* id) { + if (id == NULL || strlen(id) == 0) { stError("deleteCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeAddress) != 0){ + if (strlen(tsSnodeAddress) != 0) { return deleteRsync(id); - }else if(tsS3StreamEnabled){ + } else if (tsS3StreamEnabled) { s3DeleteObjectsByPrefix(id); } return 0; } -int deleteCheckpointFile(char* id, char* name){ +int deleteCheckpointFile(char* id, char* name) { char object[128] = {0}; snprintf(object, sizeof(object), "%s/%s", id, name); - char *tmp = object; + char* tmp = object; s3DeleteObjects((const char**)&tmp, 1); return 0; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 42280b0d0f..c9ba320f4a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -14,9 +14,9 @@ */ #include "streamInt.h" +#include "tmisce.h" #include "trpc.h" #include "ttimer.h" -#include "tmisce.h" typedef struct SBlockName { uint32_t hashValue; @@ -231,7 +231,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) buf = NULL; stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, - pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); + pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } code = 0; @@ -270,7 +270,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, initRpcMsg(&msg, TDMT_VND_STREAM_TASK_CHECK, buf, tlen + sizeof(SMsgHead)); stDebug("s-task:%s (level:%d) send check msg to s-task:0x%" PRIx64 ":0x%x (vgId:%d)", pTask->id.idStr, - pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); + pTask->info.taskLevel, pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); return 0; @@ -343,7 +343,8 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); // TODO: do not use broadcast - if (pDataBlock->info.type == STREAM_DELETE_RESULT || pDataBlock->info.type == STREAM_CHECKPOINT || pDataBlock->info.type == STREAM_TRANS_STATE) { + if (pDataBlock->info.type == STREAM_DELETE_RESULT || pDataBlock->info.type == STREAM_CHECKPOINT || + pDataBlock->info.type == STREAM_TRANS_STATE) { for (int32_t j = 0; j < numOfVgroups; j++) { code = streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]); if (code != 0) { @@ -362,7 +363,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD } code = streamSearchAndAddBlock(pTask, pReqs, pDataBlock, numOfVgroups, pDataBlock->info.id.groupId); - if(code != 0) { + if (code != 0) { destroyDispatchMsg(pReqs, numOfVgroups); return code; } @@ -376,8 +377,8 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD } static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatchMsg) { - int32_t code = 0; - int32_t msgId = pTask->execInfo.dispatch; + int32_t code = 0; + int32_t msgId = pTask->execInfo.dispatch; const char* id = pTask->id.idStr; if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { @@ -393,8 +394,8 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); - stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d vgroup(s), msgId:%d", - id, pTask->info.selfChildId, numOfVgroups, msgId); + stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d vgroup(s), msgId:%d", id, + pTask->info.selfChildId, numOfVgroups, msgId); for (int32_t i = 0; i < numOfVgroups; i++) { if (pDispatchMsg[i].blockNum > 0) { @@ -409,7 +410,8 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch } } - stDebug("s-task:%s complete shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfVgroups, msgId); + stDebug("s-task:%s complete shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfVgroups, + msgId); } return code; @@ -434,20 +436,20 @@ static void doRetryDispatchData(void* param, void* tmrId) { SArray* pList = taosArrayDup(pTask->msgInfo.pRetryList, NULL); taosArrayClear(pTask->msgInfo.pRetryList); - SStreamDispatchReq *pReq = pTask->msgInfo.pData; + SStreamDispatchReq* pReq = pTask->msgInfo.pData; if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); int32_t numOfFailed = taosArrayGetSize(pList); - stDebug("s-task:%s (child taskId:%d) retry shuffle-dispatch blocks to %d vgroup(s), msgId:%d", - id, pTask->info.selfChildId, numOfFailed, msgId); + stDebug("s-task:%s (child taskId:%d) retry shuffle-dispatch blocks to %d vgroup(s), msgId:%d", id, + pTask->info.selfChildId, numOfFailed, msgId); for (int32_t i = 0; i < numOfFailed; i++) { - int32_t vgId = *(int32_t*) taosArrayGet(pList, i); + int32_t vgId = *(int32_t*)taosArrayGet(pList, i); - for(int32_t j = 0; j < numOfVgroups; ++j) { + for (int32_t j = 0; j < numOfVgroups; ++j) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); if (pVgInfo->vgId == vgId) { stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, @@ -461,7 +463,8 @@ static void doRetryDispatchData(void* param, void* tmrId) { } } - stDebug("s-task:%s complete re-try shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfFailed, msgId); + stDebug("s-task:%s complete re-try shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, + numOfFailed, msgId); } else { int32_t vgId = pTask->outputInfo.fixedDispatcher.nodeId; SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet; @@ -478,8 +481,8 @@ static void doRetryDispatchData(void* param, void* tmrId) { if (code != TSDB_CODE_SUCCESS) { if (!streamTaskShouldStop(pTask)) { -// stDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr); -// atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); + // stDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr); + // atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); if (streamTaskShouldPause(pTask)) { streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS * 10); } else { @@ -531,10 +534,12 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } if (pDataBlock->info.parTbName[0]) { - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, + pDataBlock->info.parTbName); } else { buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, + pDataBlock->info.parTbName); } /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ @@ -576,13 +581,15 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } int32_t streamDispatchStreamBlock(SStreamTask* pTask) { - ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH)); + ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || + pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH)); const char* id = pTask->id.idStr; - int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputq.queue); + int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputq.queue); if (numOfElems > 0) { double size = SIZE_IN_MiB(taosQueueMemorySize(pTask->outputq.queue->pQueue)); - stDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id, numOfElems, size); + stDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id, + numOfElems, size); } // to make sure only one dispatch is running @@ -612,7 +619,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { int32_t code = doBuildDispatchMsg(pTask, pBlock); if (code == 0) { destroyStreamDataBlock(pBlock); - } else { // todo handle build dispatch msg failed + } else { // todo handle build dispatch msg failed } int32_t retryCount = 0; @@ -635,8 +642,9 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { if (++retryCount > MAX_CONTINUE_RETRY_COUNT) { // add to timer to retry int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s failed to dispatch msg to downstream for %d times, code:%s, add timer to retry in %dms, ref:%d", - pTask->id.idStr, retryCount, tstrerror(terrno), DISPATCH_RETRY_INTERVAL_MS, ref); + stDebug( + "s-task:%s failed to dispatch msg to downstream for %d times, code:%s, add timer to retry in %dms, ref:%d", + pTask->id.idStr, retryCount, tstrerror(terrno), DISPATCH_RETRY_INTERVAL_MS, ref); streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); break; @@ -659,7 +667,8 @@ int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) { if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { req.downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; pTask->notReadyTasks = 1; - doDispatchScanHistoryFinishMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, &pTask->outputInfo.fixedDispatcher.epSet); + doDispatchScanHistoryFinishMsg(pTask, &req, pTask->outputInfo.fixedDispatcher.nodeId, + &pTask->outputInfo.fixedDispatcher.epSet); } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgs = taosArrayGetSize(vgInfo); @@ -667,8 +676,8 @@ int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask) { char* p = NULL; streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", pTask->id.idStr, - numOfVgs, p); + stDebug("s-task:%s send scan-history data complete msg to downstream (shuffle-dispatch) %d tasks, status:%s", + pTask->id.idStr, numOfVgs, p); for (int32_t i = 0; i < numOfVgs; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); req.downstreamTaskId = pVgInfo->taskId; @@ -692,11 +701,12 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { tmsgSendReq(&pInfo->upstreamNodeEpset, &pInfo->msg); stDebug("s-task:%s level:%d checkpoint ready msg sent to upstream:0x%x", pTask->id.idStr, pTask->info.taskLevel, - pInfo->upStreamTaskId); + pInfo->upStreamTaskId); } taosArrayClear(pTask->pReadyMsgList); - stDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel, num); + stDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel, + num); return TSDB_CODE_SUCCESS; } @@ -783,7 +793,7 @@ int32_t doDispatchScanHistoryFinishMsg(SStreamTask* pTask, const SStreamScanHist char* p = NULL; streamTaskGetStatus(pTask, &p); stDebug("s-task:%s status:%s dispatch scan-history finish msg to taskId:0x%x (vgId:%d)", pTask->id.idStr, p, - pReq->downstreamTaskId, vgId); + pReq->downstreamTaskId, vgId); return 0; } @@ -866,8 +876,8 @@ int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInf return 0; } -int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, - SStreamTask* pTask, int8_t isSucceed) { +int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask, + int8_t isSucceed) { SStreamChkptReadyInfo info = {0}; buildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, isSucceed); @@ -876,7 +886,8 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa } taosArrayPush(pTask->pReadyMsgList, &info); - stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pTask->pReadyMsgList)); + stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, + (int32_t)taosArrayGetSize(pTask->pReadyMsgList)); return TSDB_CODE_SUCCESS; } @@ -927,7 +938,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, info.msg.info.noResp = 1; // refactor later. stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d", - pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index); + pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index); if (pTask->pReadyMsgList == NULL) { pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo)); @@ -937,6 +948,16 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, return 0; } +void streamClearChkptReadyMsg(SStreamTask* pTask) { + if (pTask->pReadyMsgList == NULL) return; + + for (int i = 0; i < taosArrayGetSize(pTask->pReadyMsgList); i++) { + SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, i); + rpcFreeCont(pInfo->msg.pCont); + } + taosArrayClear(pTask->pReadyMsgList); +} + int32_t tEncodeCompleteHistoryDataMsg(SEncoder* pEncoder, const SStreamCompleteHistoryMsg* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -959,7 +980,8 @@ int32_t tDecodeCompleteHistoryDataMsg(SDecoder* pDecoder, SStreamCompleteHistory return 0; } -int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, int32_t* pLen) { +int32_t streamTaskBuildScanhistoryRspMsg(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, void** pBuffer, + int32_t* pLen) { int32_t len = 0; int32_t code = 0; SEncoder encoder; @@ -1016,7 +1038,7 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, int32_t num = taosArrayGetSize(pTask->pRspMsgList); stDebug("s-task:%s add scan-history finish rsp msg for task:0x%x, total:%d", pTask->id.idStr, pReq->upstreamTaskId, - num); + num); return TSDB_CODE_SUCCESS; } @@ -1075,7 +1097,7 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId stDebug("s-task:%s downstream task:0x%x resume to normal from inputQ blocking, blocking time:%" PRId64 "ms", pTask->id.idStr, downstreamId, el); } else { - stDebug("s-task:%s dispatch completed, elapsed time:%"PRId64"ms", pTask->id.idStr, el); + stDebug("s-task:%s dispatch completed, elapsed time:%" PRId64 "ms", pTask->id.idStr, el); } // now ready for next data output @@ -1100,11 +1122,12 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { const char* id = pTask->id.idStr; int32_t vgId = pTask->pMeta->vgId; - int32_t msgId = pTask->execInfo.dispatch; + int32_t msgId = pTask->execInfo.dispatch; // follower not handle the dispatch rsp if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { - stError("s-task:%s vgId:%d is follower or task just re-launched, not handle the dispatch rsp, discard it", id, vgId); + stError("s-task:%s vgId:%d is follower or task just re-launched, not handle the dispatch rsp, discard it", id, + vgId); return TSDB_CODE_STREAM_TASK_NOT_EXIST; } @@ -1122,8 +1145,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i // flag. Here we need to retry dispatch this message to downstream task immediately. handle the case the failure // happened too fast. if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { // destination task does not exist, not retry anymore - stError("s-task:%s failed to dispatch msg to task:0x%x(vgId:%d), msgId:%d no retry, since task destroyed already", id, - pRsp->downstreamTaskId, pRsp->downstreamNodeId, msgId); + stError("s-task:%s failed to dispatch msg to task:0x%x(vgId:%d), msgId:%d no retry, since task destroyed already", + id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, msgId); } else { stError("s-task:%s failed to dispatch msgId:%d to task:0x%x(vgId:%d), code:%s, add to retry list", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, tstrerror(code)); @@ -1154,16 +1177,18 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i ASSERT(leftRsp >= 0); if (leftRsp > 0) { - stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting for %d rsp", - id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), leftRsp); + stDebug( + "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting " + "for %d rsp", + id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), leftRsp); } else { stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all rsp", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); } } else { - stDebug("s-task:%s recv fix-dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s", - id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); + stDebug("s-task:%s recv fix-dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s", id, + msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); } ASSERT(leftRsp >= 0); @@ -1185,7 +1210,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i pTask->id.idStr, DISPATCH_RETRY_INTERVAL_MS, ref); streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); - } else { // this message has been sent successfully, let's try next one. + } else { // this message has been sent successfully, let's try next one. pTask->msgInfo.retryCount = 0; // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 24228c0307..c93d658adb 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -15,11 +15,11 @@ #include "executor.h" #include "streamInt.h" +#include "streamsm.h" #include "tmisce.h" #include "tstream.h" #include "ttimer.h" #include "wal.h" -#include "streamsm.h" static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo); @@ -309,11 +309,11 @@ void tFreeStreamTask(SStreamTask* pTask) { stDebug("start to free s-task:0x%x, %p, state:%p", taskId, pTask, pTask->pState); stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64 - ", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64 - " nextProcessVer:%" PRId64", checkpointCount:%d", - taskId, pStatis->created, pStatis->init, pStatis->start, pStatis->updateCount, pStatis->latestUpdateTs, - pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, pTask->chkInfo.nextProcessVer, - pStatis->checkpoint); + ", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64 + " nextProcessVer:%" PRId64 ", checkpointCount:%d", + taskId, pStatis->created, pStatis->init, pStatis->start, pStatis->updateCount, pStatis->latestUpdateTs, + pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, pTask->chkInfo.nextProcessVer, + pStatis->checkpoint); // remove the ref by timer while (pTask->status.timerActive > 0) { @@ -358,7 +358,9 @@ void tFreeStreamTask(SStreamTask* pTask) { walCloseReader(pTask->exec.pWalReader); } + streamClearChkptReadyMsg(pTask); pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList); + if (pTask->msgInfo.pData != NULL) { destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); pTask->msgInfo.pData = NULL; @@ -422,7 +424,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->status.pSM = streamCreateStateMachine(pTask); if (pTask->status.pSM == NULL) { stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr, - tstrerror(terrno)); + tstrerror(terrno)); return terrno; } @@ -434,7 +436,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->chkInfo.checkpointVer = ver - 1; // only update when generating checkpoint pTask->chkInfo.processedVer = ver - 1; // already processed version - pTask->chkInfo.nextProcessVer = ver; // next processed version + pTask->chkInfo.nextProcessVer = ver; // next processed version pTask->dataRange.range.maxVer = ver; pTask->dataRange.range.minVer = ver; pTask->pMsgCb = pMsgCb; @@ -442,7 +444,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->outputInfo.pTokenBucket = taosMemoryCalloc(1, sizeof(STokenBucket)); if (pTask->outputInfo.pTokenBucket == NULL) { - stError("s-task:%s failed to prepare the tokenBucket, code:%s", pTask->id.idStr, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + stError("s-task:%s failed to prepare the tokenBucket, code:%s", pTask->id.idStr, + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); return TSDB_CODE_OUT_OF_MEMORY; } @@ -451,7 +454,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate, pTask->id.idStr); TdThreadMutexAttr attr = {0}; - int code = taosThreadMutexAttrInit(&attr); + int code = taosThreadMutexAttrInit(&attr); if (code != 0) { stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code)); return code; @@ -529,8 +532,8 @@ void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpS SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); if (pInfo->nodeId == nodeId) { epsetAssign(&pInfo->epSet, pEpSet); - stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, - pInfo->taskId, nodeId, buf); + stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, pInfo->taskId, + nodeId, buf); break; } } @@ -569,7 +572,7 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE if (pVgInfo->vgId == nodeId) { epsetAssign(&pVgInfo->epSet, pEpSet); stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId, - pVgInfo->taskId, nodeId, buf); + pVgInfo->taskId, nodeId, buf); break; } } @@ -578,7 +581,7 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE if (pDispatcher->nodeId == nodeId) { epsetAssign(&pDispatcher->epSet, pEpSet); stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId, - pDispatcher->taskId, nodeId, buf); + pDispatcher->taskId, nodeId, buf); } } else { // do nothing @@ -586,9 +589,9 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE } int32_t streamTaskStop(SStreamTask* pTask) { - int32_t vgId = pTask->pMeta->vgId; - int64_t st = taosGetTimestampMs(); - const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; + int64_t st = taosGetTimestampMs(); + const char* id = pTask->id.idStr; streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP); qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); @@ -635,7 +638,7 @@ int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) { p->latestUpdateTs = taosGetTimestampMs(); p->updateCount += 1; stDebug("s-task:0x%x update task nodeEp epset, updatedNodes:%d, updateCount:%d, prevTs:%" PRId64, pTask->id.taskId, - numOfNodes, p->updateCount, prevTs); + numOfNodes, p->updateCount, prevTs); for (int32_t i = 0; i < taosArrayGetSize(pNodeList); ++i) { SNodeUpdateInfo* pInfo = taosArrayGet(pNodeList, i); @@ -706,7 +709,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } - STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; + STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); if (ppStreamTask != NULL) { @@ -720,7 +723,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask) { } int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId) { - SVDropStreamTaskReq *pReq = rpcMallocCont(sizeof(SVDropStreamTaskReq)); + SVDropStreamTaskReq* pReq = rpcMallocCont(sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; From 845493294da929c49f0fa1e8eaa5834bf9feff07 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Tue, 21 Nov 2023 16:31:31 +0800 Subject: [PATCH 08/21] osfile --- include/os/osFile.h | 19 +- source/common/src/tglobal.c | 9 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmFile.c | 2 +- source/dnode/mgmt/node_util/src/dmEps.c | 2 +- source/dnode/mgmt/node_util/src/dmFile.c | 2 +- source/dnode/mnode/impl/src/mndDump.c | 2 +- source/dnode/mnode/impl/test/trans/trans1.cpp | 2 +- source/dnode/vnode/src/tsdb/tsdbFS.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFile.c | 2 +- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- source/libs/sync/src/syncRaftCfg.c | 2 +- source/libs/sync/src/syncRaftStore.c | 2 +- source/libs/wal/src/walMeta.c | 2 +- source/libs/wal/src/walWrite.c | 8 +- source/os/src/osFile.c | 707 ++++++++++++++---- source/os/test/osTests.cpp | 183 ++++- source/util/src/tlog.c | 14 +- 19 files changed, 768 insertions(+), 198 deletions(-) diff --git a/include/os/osFile.h b/include/os/osFile.h index 63483dc906..e409936468 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -54,15 +54,16 @@ extern "C" { typedef struct TdFile *TdFilePtr; -#define TD_FILE_CREATE 0x0001 -#define TD_FILE_WRITE 0x0002 -#define TD_FILE_READ 0x0004 -#define TD_FILE_TRUNC 0x0008 -#define TD_FILE_APPEND 0x0010 -#define TD_FILE_TEXT 0x0020 -#define TD_FILE_AUTO_DEL 0x0040 -#define TD_FILE_EXCL 0x0080 -#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile +#define TD_FILE_CREATE 0x0001 +#define TD_FILE_WRITE 0x0002 +#define TD_FILE_READ 0x0004 +#define TD_FILE_TRUNC 0x0008 +#define TD_FILE_APPEND 0x0010 +#define TD_FILE_TEXT 0x0020 +#define TD_FILE_AUTO_DEL 0x0040 +#define TD_FILE_EXCL 0x0080 +#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile +#define TD_FILE_WRITE_THROUGH 0x0200 TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions); TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f5df2fef21..ab8783c925 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -354,11 +354,16 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input char cfgFile[PATH_MAX + 100] = {0}; taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + char lastC = cfgDir[strlen(cfgDir) - 1]; + char *tdDirsep = TD_DIRSEP; + if (lastC == '\\' || lastC == '\/') { + tdDirsep = ""; + } if (taosIsDir(cfgDir)) { #ifdef CUS_PROMPT - snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", cfgDir, CUS_PROMPT); + snprintf(cfgFile, sizeof(cfgFile), "%s" "%s" "%s.cfg", cfgDir, tdDirsep, CUS_PROMPT); #else - snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); + snprintf(cfgFile, sizeof(cfgFile), "%s" "%s" "taos.cfg", cfgDir, tdDirsep); #endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 64e18ef06d..27baa5ede5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -169,7 +169,7 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { if (buffer == NULL) goto _OVER; terrno = 0; - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) goto _OVER; int32_t len = strlen(buffer); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index ed32e75d18..53139330a3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -200,7 +200,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { if (buffer == NULL) goto _OVER; terrno = 0; - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) goto _OVER; int32_t len = strlen(buffer); diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 3e948678a4..bee77528bd 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -232,7 +232,7 @@ int32_t dmWriteEps(SDnodeData *pData) { if (buffer == NULL) goto _OVER; terrno = 0; - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) goto _OVER; int32_t len = strlen(buffer); diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index c81efddcc1..03c6734e0c 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -120,7 +120,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) { if (buffer == NULL) goto _OVER; terrno = 0; - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) goto _OVER; int32_t len = strlen(buffer); diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 481495cbe5..5efebbc16e 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -605,7 +605,7 @@ void mndDumpSdb() { char *pCont = tjsonToString(json); int32_t contLen = strlen(pCont); char file[] = "sdb.json"; - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC| TD_FILE_WRITE_THROUGH); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to write %s since %s", file, terrstr()); diff --git a/source/dnode/mnode/impl/test/trans/trans1.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp index 92a442aa5e..aff1156449 100644 --- a/source/dnode/mnode/impl/test/trans/trans1.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -38,7 +38,7 @@ class MndTestTrans1 : public ::testing::Test { test.ServerStop(); - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); int32_t writeLen = taosWriteFile(pFile, buffer, readLen); if (writeLen < 0 || writeLen == readLen) { ASSERT(1); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index c0c74d6b87..732f46467e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -104,7 +104,7 @@ static int32_t tsdbSaveFSToFile(STsdbFS *pFS, const char *fname) { taosCalcChecksumAppend(0, pData, size); // save to file - TdFilePtr pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + TdFilePtr pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 70a83ebdbb..dd49a3319c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -85,7 +85,7 @@ static int32_t save_json(const cJSON *json, const char *fname) { char *data = cJSON_PrintUnformatted(json); if (data == NULL) return TSDB_CODE_OUT_OF_MEMORY; - TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + TdFilePtr fp = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (fp == NULL) { code = TAOS_SYSTEM_ERROR(code); goto _exit; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 62b37cd0a6..3ee0c482a7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -177,7 +177,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); // open - pFD = taosOpenFile(fname, TD_FILE_WRITE); + pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_WRITE_THROUGH); if (pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 50ca2f5d03..f9b5e9168c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -177,7 +177,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } // save info to a vnode_tmp.json - pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) { vError("failed to open info file:%s for write:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 0dcc3eee29..0e98fe94eb 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -103,7 +103,7 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { if (buffer == NULL) goto _OVER; terrno = 0; - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) goto _OVER; int32_t len = strlen(buffer); diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 051106b99d..c200c6cb4b 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -128,7 +128,7 @@ int32_t raftStoreWriteFile(SSyncNode *pNode) { if (buffer == NULL) goto _OVER; terrno = 0; - pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pFile == NULL) goto _OVER; int32_t len = strlen(buffer); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index f5e5427c68..933014466a 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -873,7 +873,7 @@ int walSaveMeta(SWal* pWal) { return -1; } - TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pMetaFile == NULL) { wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ef97bff896..a2ca953c59 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -133,7 +133,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); + TdFilePtr pIdxFile = pWal->pIdxFile; if (pIdxFile == NULL) { taosThreadMutexUnlock(&pWal->mutex); @@ -153,7 +153,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); + TdFilePtr pLogFile = pWal->pLogFile; wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr); if (pLogFile == NULL) { // TODO @@ -204,8 +204,6 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pWal->vers.lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; - taosCloseFile(&pIdxFile); - taosCloseFile(&pLogFile); code = walSaveMeta(pWal); if (code < 0) { @@ -605,7 +603,7 @@ int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SWalSync return -1; } - if (pWal->pIdxFile == NULL || pWal->pIdxFile == NULL || pWal->writeCur < 0) { + if (pWal->pIdxFile == NULL || pWal->pLogFile == NULL || pWal->writeCur < 0) { if (walInitWriteFile(pWal) < 0) { taosThreadMutexUnlock(&pWal->mutex); return -1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 30f079d10d..cf3bce1ad4 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -21,11 +21,12 @@ #include #include #include +#include #define F_OK 0 #define W_OK 2 #define R_OK 4 -#define _SEND_FILE_STEP_ 1000 +#define _SEND_FILE_STEP_ 1024 #else #include @@ -44,12 +45,22 @@ typedef int32_t FileFd; +#ifdef WINDOWS +typedef struct TdFile { + TdThreadRwlock rwlock; + int refId; + HANDLE hFile; + FILE* fp; + int32_t tdFileOptions; +} TdFile; +#else typedef struct TdFile { TdThreadRwlock rwlock; int refId; FileFd fd; FILE *fp; } TdFile; +#endif // WINDOWS #define FILE_WITH_LOCK 1 @@ -240,15 +251,12 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *a return 0; } int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { - if (pFile == NULL || pFile->fd < 0) { +#ifdef WINDOWS + if (pFile == NULL || pFile->hFile == NULL) { return -1; } - -#ifdef WINDOWS - BY_HANDLE_FILE_INFORMATION bhfi; - HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd); - if (GetFileInformationByHandle(handle, &bhfi) == FALSE) { + if (GetFileInformationByHandle(pFile->hFile, &bhfi) == FALSE) { printf("taosFStatFile get file info fail."); return -1; } @@ -262,7 +270,9 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { } #else - + if (pFile == NULL || pFile->fd < 0) { + return -1; + } struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { @@ -282,116 +292,363 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { return 0; } -TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { - int fd = -1; - FILE *fp = NULL; - if (tdFileOptions & TD_FILE_STREAM) { - char *mode = NULL; - if (tdFileOptions & TD_FILE_APPEND) { - mode = (tdFileOptions & TD_FILE_TEXT) ? "at+" : "ab+"; - } else if (tdFileOptions & TD_FILE_TRUNC) { - mode = (tdFileOptions & TD_FILE_TEXT) ? "wt+" : "wb+"; - } else if ((tdFileOptions & TD_FILE_READ) && !(tdFileOptions & TD_FILE_WRITE)) { - mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; - } else { - mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; - } - ASSERT(!(tdFileOptions & TD_FILE_EXCL)); - if (tdFileOptions & TD_FILE_EXCL) { - return NULL; - } - fp = fopen(path, mode); - if (fp == NULL) { - return NULL; - } +FILE *taosOpenFileForStream(const char *path, int32_t tdFileOptions) { + char *mode = NULL; + if (tdFileOptions & TD_FILE_APPEND) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "at+" : "ab+"; + } else if (tdFileOptions & TD_FILE_TRUNC) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt+" : "wb+"; + } else if ((tdFileOptions & TD_FILE_READ) && !(tdFileOptions & TD_FILE_WRITE)) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; } else { - int access = O_BINARY; - access |= (tdFileOptions & TD_FILE_CREATE) ? O_CREAT : 0; - if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) { - access |= O_RDWR; - } else if (tdFileOptions & TD_FILE_WRITE) { - access |= O_WRONLY; - } else if (tdFileOptions & TD_FILE_READ) { - access |= O_RDONLY; - } - access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; - access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; - access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; - access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; -#ifdef WINDOWS - int32_t pmode = _S_IREAD | _S_IWRITE; - if (tdFileOptions & TD_FILE_AUTO_DEL) { - pmode |= _O_TEMPORARY; - } - fd = _open(path, access, pmode); -#else - fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); -#endif - if (fd == -1) { - return NULL; - } + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; } - - TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); - if (pFile == NULL) { - if (fd >= 0) close(fd); - if (fp != NULL) fclose(fp); + ASSERT(!(tdFileOptions & TD_FILE_EXCL)); + if (tdFileOptions & TD_FILE_EXCL) { return NULL; } - -#if FILE_WITH_LOCK - taosThreadRwlockInit(&(pFile->rwlock), NULL); -#endif - pFile->fd = fd; - pFile->fp = fp; - pFile->refId = 0; - - if (tdFileOptions & TD_FILE_AUTO_DEL) { -#ifdef WINDOWS - // do nothing, since the property of pmode is set with _O_TEMPORARY; the OS will recycle - // the file handle, as well as the space on disk. -#else - // Remove it instantly, so when the program exits normally/abnormally, the file - // will be automatically remove by OS. - unlink(path); -#endif - } - - return pFile; + return fopen(path, mode); } -int32_t taosCloseFile(TdFilePtr *ppFile) { - int32_t code = 0; - if (ppFile == NULL || *ppFile == NULL) { +#ifdef WINDOWS +HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { + DWORD openMode = 0; + DWORD access = 0; + DWORD fileFlag = FILE_ATTRIBUTE_NORMAL; + DWORD shareMode = FILE_SHARE_READ; + + openMode = OPEN_EXISTING; + if (tdFileOptions & TD_FILE_CREATE) { + openMode = OPEN_ALWAYS; + } else if (tdFileOptions & TD_FILE_EXCL) { + openMode = CREATE_NEW; + } else if ((tdFileOptions & TD_FILE_TRUNC)) { + openMode = TRUNCATE_EXISTING; + access |= GENERIC_WRITE; + } + if (tdFileOptions & TD_FILE_APPEND) { + access |= FILE_APPEND_DATA; + } + if (tdFileOptions & TD_FILE_WRITE) { + access |= GENERIC_WRITE; + } + + shareMode |= FILE_SHARE_WRITE; + + access |= GENERIC_READ; + + if (tdFileOptions & TD_FILE_AUTO_DEL) { + fileFlag |= FILE_ATTRIBUTE_TEMPORARY; + } + if (tdFileOptions & TD_FILE_WRITE_THROUGH) { + fileFlag |= FILE_FLAG_WRITE_THROUGH; + } + + HANDLE h = CreateFile(path, access, shareMode, NULL, openMode, fileFlag, NULL); + if (h != INVALID_HANDLE_VALUE && (tdFileOptions & TD_FILE_APPEND) && (tdFileOptions & TD_FILE_WRITE)) { + SetFilePointer(h, 0, NULL, FILE_END); + } + if (h == INVALID_HANDLE_VALUE) { + DWORD dwError = GetLastError(); + LPVOID lpMsgBuf; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError, + 0, + (LPTSTR)&lpMsgBuf, 0, NULL); + printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf); + LocalFree(lpMsgBuf); + } + return h; +} + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { +#if FILE_WITH_LOCK + taosThreadRwlockRdlock(&(pFile->rwlock)); +#endif + if (pFile->hFile == NULL) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } + + DWORD bytesRead; + if (!ReadFile(pFile->hFile, buf, count, &bytesRead, NULL)) { + bytesRead = -1; + } +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return bytesRead; +} + +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + if (pFile == NULL || pFile->hFile == NULL) { return 0; } #if FILE_WITH_LOCK - taosThreadRwlockWrlock(&((*ppFile)->rwlock)); + taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - if ((*ppFile)->fp != NULL) { - fflush((*ppFile)->fp); - fclose((*ppFile)->fp); - (*ppFile)->fp = NULL; + + DWORD bytesWritten; + if (!WriteFile(pFile->hFile, buf, count, &bytesWritten, NULL)) { + bytesWritten = -1; } - if ((*ppFile)->fd >= 0) { -#ifdef WINDOWS - HANDLE h = (HANDLE)_get_osfhandle((*ppFile)->fd); - !FlushFileBuffers(h); -#else - // warning: never fsync silently in base lib - /*fsync((*ppFile)->fd);*/ -#endif - code = close((*ppFile)->fd); - (*ppFile)->fd = -1; - } - (*ppFile)->refId = 0; + #if FILE_WITH_LOCK - taosThreadRwlockUnlock(&((*ppFile)->rwlock)); - taosThreadRwlockDestroy(&((*ppFile)->rwlock)); + taosThreadRwlockUnlock(&(pFile->rwlock)); #endif - taosMemoryFree(*ppFile); - *ppFile = NULL; - return code; + return bytesWritten; +} + +int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { + if (pFile == NULL) { + return 0; + } +#if FILE_WITH_LOCK + taosThreadRwlockWrlock(&(pFile->rwlock)); +#endif + ASSERT(pFile->hFile != NULL); // Please check if you have closed the file. + if (pFile->hFile == NULL) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return 0; + } + + DWORD ret = 0; + OVERLAPPED ol = {0}; + ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20); + ol.Offset = (uint32_t)(offset & 0xFFFFFFFFLL); + + SetLastError(0); + BOOL result = WriteFile(pFile->hFile, buf, count, &ret, &ol); + if (!result) { + errno = GetLastError(); + ret = -1; + } + +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return ret; +} + +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL || pFile->hFile == NULL) { + return -1; + } +#if FILE_WITH_LOCK + taosThreadRwlockRdlock(&(pFile->rwlock)); +#endif + + LARGE_INTEGER liOffset; + liOffset.QuadPart = offset; + if (!SetFilePointerEx(pFile->hFile, liOffset, NULL, whence)) { + return -1; + } + + liOffset.QuadPart = 0; + if (!SetFilePointerEx(pFile->hFile, liOffset, &liOffset, FILE_CURRENT)) { + return -1; + } +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return liOffset.QuadPart; +} + +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { + if (pFile == NULL || pFile->hFile == NULL) { + return 0; + } + + if (size != NULL) { + LARGE_INTEGER fileSize; + if (!GetFileSizeEx(pFile->hFile, &fileSize)) { + return -1; // Error getting file size + } + *size = fileSize.QuadPart; + } + + if (mtime != NULL) { + FILETIME creationTime, lastAccessTime, lastWriteTime; + if (!GetFileTime(pFile->hFile, &creationTime, &lastAccessTime, &lastWriteTime)) { + return -1; // Error getting file time + } + // Convert the FILETIME structure to a time_t value + ULARGE_INTEGER ull; + ull.LowPart = lastWriteTime.dwLowDateTime; + ull.HighPart = lastWriteTime.dwHighDateTime; + *mtime = (int32_t)((ull.QuadPart - 116444736000000000ULL) / 10000000ULL); + } + return 0; +} + +int32_t taosLockFile(TdFilePtr pFile) { + if (pFile == NULL || pFile->hFile == NULL) { + return -1; + } + + BOOL fSuccess = FALSE; + LARGE_INTEGER fileSize; + OVERLAPPED overlapped = {0}; + + fSuccess = LockFileEx(pFile->hFile, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, // reserved + ~0, // number of bytes to lock low + ~0, // number of bytes to lock high + &overlapped // overlapped structure + ); + if (!fSuccess) { + return GetLastError(); + } + return 0; +} + +int32_t taosUnLockFile(TdFilePtr pFile) { + if (pFile == NULL || pFile->hFile == NULL) { + return 0; + } + BOOL fSuccess = FALSE; + OVERLAPPED overlapped = {0}; + + fSuccess = UnlockFileEx(pFile->hFile, 0, ~0, ~0, &overlapped); + if (!fSuccess) { + return GetLastError(); + } + return 0; +} + +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { + if (pFile == NULL) { + return 0; + } + if (pFile->hFile == NULL) { + printf("Ftruncate file error, hFile was null\n"); + return -1; + } + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(pFile->hFile, li_0, NULL, FILE_CURRENT); + if (!cur) { + printf("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(pFile->hFile, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + printf("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(pFile->hFile)) { + int error = GetLastError(); + printf("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + return 0; +} + +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + if (pFileOut == NULL || pFileIn == NULL) { + return 0; + } + if (pFileIn->hFile == NULL || pFileOut->hFile == NULL) { + return 0; + } + + LARGE_INTEGER fileOffset; + fileOffset.QuadPart = *offset; + + if (!SetFilePointerEx(pFileIn->hFile, fileOffset, &fileOffset, FILE_BEGIN)) { + return -1; + } + + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + + DWORD bytesRead; + DWORD bytesWritten; + for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + if (!ReadFile(pFileIn->hFile, buffer, _SEND_FILE_STEP_, &bytesRead, NULL)) { + return writeLen; + } + + if (bytesRead <= 0) { + return writeLen; + } else if (bytesRead < _SEND_FILE_STEP_) { + if (!WriteFile(pFileOut->hFile, buffer, bytesRead, &bytesWritten, NULL)) { + return -1; + } else { + return (int64_t)(writeLen + bytesRead); + } + } else { + if (!WriteFile(pFileOut->hFile, buffer, _SEND_FILE_STEP_, &bytesWritten, NULL)) { + return -1; + } else { + writeLen += _SEND_FILE_STEP_; + } + } + } + + int64_t remain = size - writeLen; + if (remain > 0) { + DWORD bytesRead; + if (!ReadFile(pFileIn->hFile, buffer, (DWORD)remain, &bytesRead, NULL)) { + return -1; + } + + if (bytesRead <= 0) { + return writeLen; + } else { + DWORD bytesWritten; + if (!WriteFile(pFileOut->hFile, buffer, bytesRead, &bytesWritten, NULL)) { + return -1; + } else { + writeLen += bytesWritten; + } + } + } + return writeLen; +} + +#else +int taosOpenFileNotStream(const char *path, int32_t tdFileOptions) { + int access = O_BINARY; + access |= (tdFileOptions & TD_FILE_CREATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) { + access |= O_RDWR; + } else if (tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + } else if (tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + return fd; } int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { @@ -407,7 +664,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { } int64_t leftbytes = count; int64_t readbytes; - char *tbuf = (char *)buf; + char * tbuf = (char *)buf; while (leftbytes > 0) { #ifdef WINDOWS @@ -441,42 +698,6 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { return count; } -int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { - if (pFile == NULL) { - return 0; - } -#if FILE_WITH_LOCK - taosThreadRwlockRdlock(&(pFile->rwlock)); -#endif - ASSERT(pFile->fd >= 0); // Please check if you have closed the file. - if (pFile->fd < 0) { -#if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); -#endif - return -1; - } -#ifdef WINDOWS - DWORD ret = 0; - OVERLAPPED ol = {0}; - ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20); - ol.Offset = (uint32_t)(offset & 0xFFFFFFFFLL); - - HANDLE handle = (HANDLE)_get_osfhandle(pFile->fd); - SetLastError(0); - BOOL result = ReadFile(handle, buf, count, &ret, &ol); - if (!result && GetLastError() != ERROR_HANDLE_EOF) { - errno = GetLastError(); - ret = -1; - } -#else - int64_t ret = pread(pFile->fd, buf, count, offset); -#endif -#if FILE_WITH_LOCK - taosThreadRwlockUnlock(&(pFile->rwlock)); -#endif - return ret; -} - int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { if (pFile == NULL) { return 0; @@ -493,7 +714,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { int64_t nleft = count; int64_t nwritten = 0; - char *tbuf = (char *)buf; + char * tbuf = (char *)buf; while (nleft > 0) { nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); @@ -706,25 +927,6 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { #endif } -int32_t taosFsyncFile(TdFilePtr pFile) { - if (pFile == NULL) { - return 0; - } - - // this implementation is WRONG - // fflush is not a replacement of fsync - if (pFile->fp != NULL) return fflush(pFile->fp); - if (pFile->fd >= 0) { -#ifdef WINDOWS - HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); - return !FlushFileBuffers(h); -#else - return fsync(pFile->fd); -#endif - } - return 0; -} - int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { if (pFileOut == NULL || pFileIn == NULL) { return 0; @@ -824,6 +1026,167 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in #endif } +#endif // WINDOWS + +TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { + FILE *fp = NULL; +#ifdef WINDOWS + HANDLE hFile = NULL; +#else + int fd = -1; +#endif + if (tdFileOptions & TD_FILE_STREAM) { + fp = taosOpenFileForStream(path, tdFileOptions); + if (fp == NULL) return NULL; + } else { +#ifdef WINDOWS + hFile = taosOpenFileNotStream(path, tdFileOptions); + if (hFile == INVALID_HANDLE_VALUE) return NULL; +#else + fd = taosOpenFileNotStream(path, tdFileOptions); + if (fd == -1) return NULL; +#endif + } + + TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); + if (pFile == NULL) { +#ifdef WINDOWS + if (hFile != NULL) CloseHandle(hFile); +#else + if (fd >= 0) close(fd); +#endif + if (fp != NULL) fclose(fp); + return NULL; + } + +#if FILE_WITH_LOCK + taosThreadRwlockInit(&(pFile->rwlock), NULL); +#endif + pFile->fp = fp; + pFile->refId = 0; + + #ifdef WINDOWS + pFile->hFile = hFile; + pFile->tdFileOptions = tdFileOptions; + // do nothing, since the property of pmode is set with _O_TEMPORARY; the OS will recycle + // the file handle, as well as the space on disk. +#else + pFile->fd = fd; + // Remove it instantly, so when the program exits normally/abnormally, the file + // will be automatically remove by OS. + if (tdFileOptions & TD_FILE_AUTO_DEL) { + unlink(path); + } +#endif + return pFile; +} + +int32_t taosCloseFile(TdFilePtr *ppFile) { + int32_t code = 0; + if (ppFile == NULL || *ppFile == NULL) { + return 0; + } +#if FILE_WITH_LOCK + taosThreadRwlockWrlock(&((*ppFile)->rwlock)); +#endif + if ((*ppFile)->fp != NULL) { + fflush((*ppFile)->fp); + fclose((*ppFile)->fp); + (*ppFile)->fp = NULL; + } +#ifdef WINDOWS + if ((*ppFile)->hFile != NULL) { + // FlushFileBuffers((*ppFile)->hFile); + if (!CloseHandle((*ppFile)->hFile)) { + code = -1; + } + (*ppFile)->hFile = NULL; +#else + if ((*ppFile)->fd >= 0) { + // warning: never fsync silently in base lib + /*fsync((*ppFile)->fd);*/ + code = close((*ppFile)->fd); + (*ppFile)->fd = -1; +#endif + } + (*ppFile)->refId = 0; +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&((*ppFile)->rwlock)); + taosThreadRwlockDestroy(&((*ppFile)->rwlock)); +#endif + taosMemoryFree(*ppFile); + *ppFile = NULL; + return code; +} + +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { + if (pFile == NULL) { + return 0; + } + +#ifdef WINDOWS +#if FILE_WITH_LOCK + taosThreadRwlockRdlock(&(pFile->rwlock)); +#endif + ASSERT(pFile->hFile != NULL); // Please check if you have closed the file. + if (pFile->hFile == NULL) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } + DWORD ret = 0; + OVERLAPPED ol = {0}; + ol.OffsetHigh = (uint32_t)((offset & 0xFFFFFFFF00000000LL) >> 0x20); + ol.Offset = (uint32_t)(offset & 0xFFFFFFFFLL); + + SetLastError(0); + BOOL result = ReadFile(pFile->hFile, buf, count, &ret, &ol); + if (!result && GetLastError() != ERROR_HANDLE_EOF) { + errno = GetLastError(); + ret = -1; + } +#else +#if FILE_WITH_LOCK + taosThreadRwlockRdlock(&(pFile->rwlock)); +#endif + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } + int64_t ret = pread(pFile->fd, buf, count, offset); +#endif +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return ret; +} + +int32_t taosFsyncFile(TdFilePtr pFile) { + if (pFile == NULL) { + return 0; + } + + // this implementation is WRONG + // fflush is not a replacement of fsync + if (pFile->fp != NULL) return fflush(pFile->fp); +#ifdef WINDOWS + if (pFile->hFile != NULL) { + if (pFile->tdFileOptions & TD_FILE_WRITE_THROUGH) { + return 0; + } + return !FlushFileBuffers(pFile->hFile); +#else + if (pFile->fd >= 0) { + return fsync(pFile->fd); +#endif + } + return 0; +} + void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { if (pFile == NULL || pFile->fp == NULL) { return; @@ -834,7 +1197,13 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { va_end(ap); } -bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; } +bool taosValidFile(TdFilePtr pFile) { +#ifdef WINDOWS + return pFile != NULL && pFile->hFile != NULL; +#else + return pFile != NULL && pFile->fd > 0; +#endif +} int32_t taosUmaskFile(int32_t maskVal) { #ifdef WINDOWS @@ -960,14 +1329,20 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { goto cmp_end; } - pFile = taosOpenFile(destFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pFile == NULL) { + int access = O_BINARY | O_WRONLY | O_TRUNC | O_CREAT; +#ifdef WINDOWS + int32_t pmode = _S_IREAD | _S_IWRITE; +#else + int32_t pmode = S_IRWXU | S_IRWXG | S_IRWXO; +#endif + int fd = open(destFileName, access, pmode); + if (fd < 0) { ret = -2; goto cmp_end; } // Both gzclose() and fclose() will close the associated fd, so they need to have different fds. - FileFd gzFd = dup(pFile->fd); + FileFd gzFd = dup(fd); if (gzFd < 0) { ret = -4; goto cmp_end; diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index a2ccc4de02..e2185aeac2 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -15,6 +15,7 @@ #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -29,6 +30,10 @@ #include "os.h" #include "tlog.h" +#ifdef WINDOWS +#include +#endif // WINDOWS + TEST(osTest, osSystem) { const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; @@ -68,7 +73,8 @@ void fileOperateOnBusy(void *param) { char * fname = (char *)param; TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); printf("On busy thread open file\n"); - ASSERT_NE(pFile, nullptr); + if (pFile == NULL) return; + // ASSERT_NE(pFile, nullptr); int ret = taosLockFile(pFile); printf("On busy thread lock file ret:%d\n", ret); @@ -97,6 +103,7 @@ TEST(osTest, osFile) { TdFilePtr pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); ASSERT_NE(pOutFD, nullptr); printf("create file success\n"); + taosCloseFile(&pOutFD); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); printf("open file\n"); @@ -135,4 +142,178 @@ TEST(osTest, osFile) { //printf("remove file success"); } +#ifndef OSFILE_PERFORMANCE_TEST + +#define MAX_WORDS 100 +#define MAX_WORD_LENGTH 20 +#define MAX_TEST_FILE_SIZE 100000 +#define TESTTIMES 1000 + +char *getRandomWord() { + static char words[][MAX_WORD_LENGTH] = { + "Lorem", "ipsum", "dolor", "sit", "amet", "consectetur", "adipiscing", "elit", + "sed", "do", "eiusmod", "tempor", "incididunt", "ut", "labore", "et", "dolore", "magna", + "aliqua", "Ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco", + "Why", "do", "programmers", "prefer", "using", "dark", "mode?", "Because", "light", "attracts", + "bugs", "and", "they", "want", "to", "code", "in", "peace,", "like", "a", "ninja", "in", "the", "shadows." + "aliqua", "Ut", "enim", "ad", "minim", "veniam", "quis", "nostrud", "exercitation", "ullamco", + "laboris", "nisi", "ut", "aliquip", "ex", "ea", "commodo", "consequat", "Duis", "aute", "irure", + "dolor", "in", "reprehenderit", "in", "voluptate", "velit", "esse", "cillum", "dolore", "eu", + "fugiat", "nulla", "pariatur", "Excepteur", "sint", "occaecat", "cupidatat", "non", "proident", + "sunt", "in", "culpa", "qui", "officia", "deserunt", "mollit", "anim", "id", "est", "laborum" + }; + + return words[taosRand() % MAX_WORDS]; +} + +int64_t fillBufferWithRandomWords(char *buffer, int64_t maxBufferSize) { + int64_t len = 0; + while (len < maxBufferSize) { + char * word = getRandomWord(); + size_t wordLen = strlen(word); + + if (len + wordLen + 1 < maxBufferSize) { + strcat(buffer, word); + strcat(buffer, " "); + len += wordLen + 1; + } else { + break; + } + } + return len; +} + +int64_t calculateAverage(int64_t arr[], int size) { + int64_t sum = 0; + for (int i = 0; i < size; i++) { + sum += arr[i]; + } + return sum / size; +} + +int64_t calculateMax(int64_t arr[], int size) { + int64_t max = arr[0]; + for (int i = 1; i < size; i++) { + if (arr[i] > max) { + max = arr[i]; + } + } + return max; +} + +int64_t calculateMin(int64_t arr[], int size) { + int64_t min = arr[0]; + for (int i = 1; i < size; i++) { + if (arr[i] < min) { + min = arr[i]; + } + } + return min; +} + +TEST(osTest, osFilePerformance) { + printf("os file performance testting...\n"); + int64_t WriteFileCost; + int64_t ReadFileCost; + int64_t OpenForWriteCloseFileCost; + int64_t OpenForReadCloseFileCost; + + char * buffer; + char * writeBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE); + char * readBuffer = (char *)taosMemoryCalloc(1, MAX_TEST_FILE_SIZE); + int64_t size = fillBufferWithRandomWords(writeBuffer, MAX_TEST_FILE_SIZE); + char * fname = "./osFilePerformanceTest.txt"; + + TdFilePtr pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + ASSERT_NE(pOutFD, nullptr); + taosCloseFile(&pOutFD); + + printf("os file performance start write...\n"); + int64_t t1 = taosGetTimestampUs(); + for (int i = 0; i < TESTTIMES; ++i) { + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_WRITE_THROUGH); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, writeBuffer, size); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + } + + int64_t t2 = taosGetTimestampUs(); + WriteFileCost = t2 - t1; + + printf("os file performance start read...\n"); + for (int i = 0; i < TESTTIMES; ++i) { + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + ASSERT_NE(pFile, nullptr); + taosReadFile(pFile, readBuffer, size); + taosCloseFile(&pFile); + int readLine = strlen(readBuffer); + ASSERT_EQ(size, readLine); + } + int64_t t3 = taosGetTimestampUs(); + ReadFileCost = t3 - t2; + + printf("os file performance start open1...\n"); + for (int i = 0; i < TESTTIMES; ++i) { + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE); + ASSERT_NE(pFile, nullptr); + taosCloseFile(&pFile); + } + int64_t t4 = taosGetTimestampUs(); + OpenForWriteCloseFileCost = t4 - t3; + + printf("os file performance start open2...\n"); + for (int i = 0; i < TESTTIMES; ++i) { + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_READ); + ASSERT_NE(pFile, nullptr); + taosCloseFile(&pFile); + } + int64_t t5 = taosGetTimestampUs(); + OpenForReadCloseFileCost = t5 - t4; + +#ifdef WINDOWS + printf("os file performance start window native...\n"); + for (int i = 0; i < TESTTIMES; ++i) { + HANDLE hFile = CreateFile(fname, // 文件名 + GENERIC_WRITE, // 写权限 + FILE_SHARE_READ, // 不共享 + NULL, // 默认安全描述符 + OPEN_ALWAYS, // 打开已存在的文件 + FILE_FLAG_WRITE_THROUGH, // 文件标志，可以根据实际需求调整 + NULL // 模板文件句柄，对于创建新文件不需要 + ); + + if (hFile == INVALID_HANDLE_VALUE) { + printf("Error opening file\n"); + break; + } + + // 写入数据 + DWORD bytesWritten; + if (!WriteFile(hFile, writeBuffer, size, &bytesWritten, NULL)) { + // 处理错误 + printf("Error writing to file\n"); + CloseHandle(hFile); + break; + } + // 关闭文件 + CloseHandle(hFile); + } + int64_t t6 = taosGetTimestampUs(); + int64_t nativeWritCost = t6 - t5; + + printf("Test Write file using native API %d times, cost: %" PRId64 "us\n", TESTTIMES, nativeWritCost); +#endif // WINDOWS + + taosMemoryFree(writeBuffer); + taosMemoryFree(readBuffer); + + printf("Test Write file %d times, cost: %" PRId64 "us\n", TESTTIMES, WriteFileCost); + printf("Test Read file %d times, cost: %" PRId64 "us\n", TESTTIMES, ReadFileCost); + printf("Test OpenForWrite & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForWriteCloseFileCost); + printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost); +} + +#endif OSFILE_PERFORMANCE_TEST + #pragma GCC diagnostic pop diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index aa6719f604..a79d387234 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -153,7 +153,12 @@ int32_t taosInitSlowLog() { #endif if (strlen(tsLogDir) != 0) { - snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); + char lastC = tsLogDir[strlen(tsLogDir) - 1]; + if (lastC == '\\' || lastC == '\/') { + snprintf(fullName, PATH_MAX, "%s" "%s", tsLogDir, logFileName); + } else { + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); + } } else { snprintf(fullName, PATH_MAX, "%s", logFileName); } @@ -177,7 +182,12 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { char fullName[PATH_MAX] = {0}; if (strlen(tsLogDir) != 0) { - snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); + char lastC = tsLogDir[strlen(tsLogDir) - 1]; + if (lastC == '\\' || lastC == '\/') { + snprintf(fullName, PATH_MAX, "%s" "%s", tsLogDir, logName); + } else { + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); + } } else { snprintf(fullName, PATH_MAX, "%s", logName); } From 01768194616484ac40ea0673a9d9a1c8f461028d Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Tue, 21 Nov 2023 16:49:19 +0800 Subject: [PATCH 09/21] fix linux build failed --- source/common/src/tglobal.c | 2 +- source/util/src/tlog.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index ab8783c925..27872d140a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -356,7 +356,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); char lastC = cfgDir[strlen(cfgDir) - 1]; char *tdDirsep = TD_DIRSEP; - if (lastC == '\\' || lastC == '\/') { + if (lastC == '\\' || lastC == '/') { tdDirsep = ""; } if (taosIsDir(cfgDir)) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a79d387234..fead2f1040 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -154,7 +154,7 @@ int32_t taosInitSlowLog() { if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; - if (lastC == '\\' || lastC == '\/') { + if (lastC == '\\' || lastC == '/') { snprintf(fullName, PATH_MAX, "%s" "%s", tsLogDir, logFileName); } else { snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); @@ -183,7 +183,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { char fullName[PATH_MAX] = {0}; if (strlen(tsLogDir) != 0) { char lastC = tsLogDir[strlen(tsLogDir) - 1]; - if (lastC == '\\' || lastC == '\/') { + if (lastC == '\\' || lastC == '/') { snprintf(fullName, PATH_MAX, "%s" "%s", tsLogDir, logName); } else { snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); From 916f5d4cf9d1d54223a0079526b6db2fca85e959 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 21 Nov 2023 19:15:57 +0800 Subject: [PATCH 10/21] fix:format-overflow may be happend --- utils/test/c/tmq_taosx_ci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 7dbde60632..8a7074844a 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -623,8 +623,8 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void initLogFile() { - char f1[256] = {0}; - char f2[256] = {0}; + char f1[1024] = {0}; + char f2[1024] = {0}; if (g_conf.snapShot) { sprintf(f1, "%s/../log/tmq_taosx_tmp_snapshot.source", g_conf.dir); From 45170a73b3da032ef5a76dc749446433fc83705a Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Tue, 21 Nov 2023 19:54:33 +0800 Subject: [PATCH 11/21] close file before create in rollback --- source/libs/wal/src/walWrite.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a2ca953c59..33d8d34514 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -133,7 +133,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - TdFilePtr pIdxFile = pWal->pIdxFile; + taosCloseFile(&pWal->pIdxFile); + TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); if (pIdxFile == NULL) { taosThreadMutexUnlock(&pWal->mutex); @@ -153,7 +154,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - TdFilePtr pLogFile = pWal->pLogFile; + taosCloseFile(&pWal->pLogFile); + TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr); if (pLogFile == NULL) { // TODO @@ -205,6 +207,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + taosCloseFile(&pIdxFile); + taosCloseFile(&pLogFile); + code = walSaveMeta(pWal); if (code < 0) { wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); From 45d7e9f02b9751bb959576fe20fca84865cc79d8 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 22 Nov 2023 10:47:10 +0800 Subject: [PATCH 12/21] fix: goto error --- source/libs/index/src/indexFilter.c | 1 + source/libs/scalar/src/scalar.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index bfdcd2b030..7ed36fbf9e 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -328,6 +328,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type)); if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); + param->pFilter = NULL; indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 3e003234cf..90cec4522f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -208,6 +208,7 @@ void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); taosMemoryFreeClear(param->columnData); + param->columnData = NULL; } if (param->pHashFilter != NULL) { @@ -845,6 +846,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SScalarParam *params = NULL; int32_t rowNum = 0; int32_t code = 0; + int32_t paramNum = 0; // json not support in in operator if (nodeType(node->pLeft) == QUERY_NODE_VALUE) { @@ -865,7 +867,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); - int32_t paramNum = scalarGetOperatorParamNum(node->opType); + paramNum = scalarGetOperatorParamNum(node->opType); SScalarParam *pLeft = ¶ms[0]; SScalarParam *pRight = paramNum > 1 ? ¶ms[1] : NULL; From b35c971278d880f35cb5831a2f81ea94ea32e937 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Nov 2023 17:29:19 +0800 Subject: [PATCH 13/21] fix(stream):remove unused function. --- include/libs/stream/tstream.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6e191e412d..d48665d351 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -787,7 +787,6 @@ int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); int32_t streamRestoreParam(SStreamTask* pTask); void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); void streamTaskResume(SStreamTask* pTask); -void streamTaskEnablePause(SStreamTask* pTask); int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet); From d88b32e93ddbc28e209459da09c7743be281455e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Nov 2023 18:18:43 +0800 Subject: [PATCH 14/21] fix(stream): fix memory leak. --- source/dnode/mnode/impl/src/mndStream.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index eca84ab2be..0eec204719 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -145,6 +145,7 @@ int32_t mndInitStream(SMnode *pMnode) { void mndCleanupStream(SMnode *pMnode) { taosArrayDestroy(execInfo.pTaskList); taosHashCleanup(execInfo.pTaskMap); + taosHashCleanup(execInfo.transMgmt.pDBTrans); taosThreadMutexDestroy(&execInfo.lock); mDebug("mnd stream exec info cleanup"); } From b7342d53dfe4edeaa364962557b558226ed7ab38 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Wed, 22 Nov 2023 19:37:08 +0800 Subject: [PATCH 15/21] return DEAL_RES_ERROR; --- source/libs/parser/src/parTranslater.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3e77feb0c1..5cde5b0883 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1433,16 +1433,18 @@ static int32_t dataTypeComp(const SDataType* l, const SDataType* r) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (isMultiResFunc(pOp->pLeft)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + return DEAL_RES_ERROR; } if (isMultiResFunc(pOp->pRight)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + return DEAL_RES_ERROR; } int32_t res = scalarGetOperatorResultType(pOp); if (TSDB_CODE_SUCCESS != res) { pCxt->errCode = res; - return DEAL_RES_CONTINUE; + return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; From 44866b7b124dc969ab3ecc523b77f7b9b2cc6ae4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 22 Nov 2023 20:35:18 +0800 Subject: [PATCH 16/21] fix: possible race condition coredump --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index dd49a3319c..c9ecff8890 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -1186,11 +1186,6 @@ const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"}; static int32_t tsdbFSRunBgTask(void *arg) { STFSBgTask *task = (STFSBgTask *)arg; STFileSystem *fs = task->fs; - STFileSet *fset; - - tsdbFSGetFSet(fs, task->fid, &fset); - - ASSERT(fset != NULL && fset->bgTaskRunning == task); task->launchTime = taosGetTimestampMs(); task->run(task->arg); @@ -1203,6 +1198,10 @@ static int32_t tsdbFSRunBgTask(void *arg) { taosThreadMutexLock(&fs->tsdb->mutex); + STFileSet *fset = NULL; + tsdbFSGetFSet(fs, task->fid, &fset); + ASSERT(fset != NULL && fset->bgTaskRunning == task); + // free last tsdbDoDoneBgTask(fs, task); fset->bgTaskRunning = NULL; From 098a3d264af223dda8d4059992810fdf8d598bfe Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 22 Nov 2023 20:45:15 +0800 Subject: [PATCH 17/21] fix: varbinary sma coredump --- source/common/src/tdataformat.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d220da0d84..f97f9c0c11 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -3590,9 +3590,9 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON - NULL, // TSDB_DATA_TYPE_VARBINARY - NULL, // TSDB_DATA_TYPE_DECIMAL - NULL, // TSDB_DATA_TYPE_BLOB + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARBINARY + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_DECIMAL + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_BLOB NULL, // TSDB_DATA_TYPE_MEDIUMBLOB tColDataCalcSMAVarType // TSDB_DATA_TYPE_GEOMETRY }; From 4ea49fe28578c225a1a3f3961eac669e96cd2272 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 22 Nov 2023 23:49:16 +0800 Subject: [PATCH 18/21] fix(stream): fix the race condition in dispatching data. --- include/libs/stream/tstream.h | 1 + source/libs/stream/src/streamCheckpoint.c | 2 ++ source/libs/stream/src/streamDispatch.c | 36 +++++++---------------- source/libs/stream/src/streamExec.c | 1 + 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index d48665d351..8277d4749e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -305,6 +305,7 @@ typedef struct SCheckpointInfo { int64_t processedVer; // already processed ver, that has generated results version. int64_t nextProcessVer; // current offset in WAL, not serialize it int64_t failedId; // record the latest failed checkpoint id + bool dispatchCheckpointTrigger; } SCheckpointInfo; typedef struct SStreamStatus { diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 6201329b95..031bb812de 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -158,6 +158,7 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); if (code == 0) { + ASSERT(pTask->chkInfo.dispatchCheckpointTrigger == false); streamDispatchStreamBlock(pTask); } else { stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code)); @@ -278,6 +279,7 @@ void streamTaskClearCheckInfo(SStreamTask* pTask) { pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->checkpointNotReadyTasks = 0; pTask->checkpointAlignCnt = 0; + pTask->chkInfo.dispatchCheckpointTrigger = false; streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1f9b21becc..1306d9f746 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -593,6 +593,12 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return 0; } + if (pTask->chkInfo.dispatchCheckpointTrigger) { + stDebug("s-task:%s already send checkpoint trigger, not dispatch anymore", id); + atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); + return 0; + } + ASSERT(pTask->msgInfo.pData == NULL); stDebug("s-task:%s start to dispatch msg, set output status:%d", id, pTask->outputq.status); @@ -1039,30 +1045,14 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) { return 0; } -static void dispatchDataInFuture(void* param, void* tmrId) { - SStreamTask* pTask = param; - if (streamTaskShouldStop(pTask)) { - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s should stop, abort from timer, ref:%d", pTask->id.idStr, ref); - return; - } - - ETaskStatus status = streamTaskGetStatus(pTask, NULL); - if (status == TASK_STATUS__CK) { - stDebug("s-task:%s in checkpoint status, wait for 500ms to dispatch data downstream", pTask->id.idStr); - taosTmrReset(dispatchDataInFuture, 500, pTask, streamEnv.timer, &pTask->msgInfo.pTimer); - } else { - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s start to dispatch data, jump out of timer, ref:%d", pTask->id.idStr, ref); - streamDispatchStreamBlock(pTask); - } -} - // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); + if (delayDispatch) { + pTask->chkInfo.dispatchCheckpointTrigger = true; + } pTask->msgInfo.pData = NULL; pTask->msgInfo.dispatchMsgType = 0; @@ -1083,13 +1073,7 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId // otherwise, continue dispatch the first block to down stream task in pipeline if (delayDispatch) { - int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s in checkpoint status, add in timer, try dispatch data in 500ms, ref:%d", pTask->id.idStr, ref); - if (pTask->msgInfo.pTimer != NULL) { - taosTmrReset(dispatchDataInFuture, 500, pTask, streamEnv.timer, &pTask->msgInfo.pTimer); - } else { - pTask->msgInfo.pTimer = taosTmrStart(dispatchDataInFuture, 500, pTask, streamEnv.timer); - } + return 0; } else { streamDispatchStreamBlock(pTask); } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cae537a860..43875319b7 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -48,6 +48,7 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl return code; } + // checkpoint trigger will be checked streamDispatchStreamBlock(pTask); } From 0a4fac77049b237a6458e97da3b0bbc3328e159f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 23 Nov 2023 09:37:37 +0800 Subject: [PATCH 19/21] enh: not allow to acquire a vnode in failed mode from vmAcquireVnode --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 21 ++++++------- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 33 +++++++++++++++++++-- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 4 +-- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 34f2b5c446..9e3039d73f 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -84,6 +84,7 @@ typedef struct { } SVnodeThread; // vmInt.c +int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId); int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId); SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId); void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 64bf875a8e..d673da4be2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -282,7 +282,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { vmGenerateWrapperCfg(pMgmt, &req, &wrapperCfg); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode != NULL && !pVnode->failed) { + if (pVnode != NULL) { dError("vgId:%d, already exist", req.vgId); tFreeSCreateVnodeReq(&req); vmReleaseVnode(pMgmt, pVnode); @@ -291,10 +291,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } - ASSERT(pVnode == NULL || pVnode->failed); - - wrapperCfg.diskPrimary = pVnode ? pVnode->diskPrimary : vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId); - int32_t diskPrimary = wrapperCfg.diskPrimary; + int32_t diskPrimary = vmGetPrimaryDisk(pMgmt, vnodeCfg.vgId); + if (diskPrimary < 0) { + diskPrimary = vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId); + } + wrapperCfg.diskPrimary = diskPrimary; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); @@ -371,7 +372,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { TMSG_INFO(pMsg->msgType)); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode == NULL || pVnode->failed) { + if (pVnode == NULL) { dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; if (pVnode) vmReleaseVnode(pMgmt, pVnode); @@ -489,7 +490,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { req.vgId, TMSG_INFO(pMsg->msgType)); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode == NULL || pVnode->failed) { + if (pVnode == NULL) { dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; if (pVnode) vmReleaseVnode(pMgmt, pVnode); @@ -532,7 +533,7 @@ int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, vnode write disable:%d", req.vgId, req.disable); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); - if (pVnode == NULL || pVnode->failed) { + if (pVnode == NULL) { dError("vgId:%d, failed to disable write since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; if (pVnode) vmReleaseVnode(pMgmt, pVnode); @@ -565,7 +566,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd, req.dstVgId); pVnode = vmAcquireVnode(pMgmt, srcVgId); - if (pVnode == NULL || pVnode->failed) { + if (pVnode == NULL) { dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; if (pVnode) vmReleaseVnode(pMgmt, pVnode); @@ -680,7 +681,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); - if (pVnode == NULL || pVnode->failed) { + if (pVnode == NULL) { dError("vgId:%d, failed to alter replica since %s", vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; if (pVnode) vmReleaseVnode(pMgmt, pVnode); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 21b791eb4d..7cb31b6e5d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -19,6 +19,19 @@ #include "vnd.h" #include "libs/function/tudf.h" +int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { + int32_t diskId = -1; + SVnodeObj *pVnode = NULL; + + taosThreadRwlockRdlock(&pMgmt->lock); + taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); + if (pVnode != NULL) { + diskId = pVnode->diskPrimary; + } + taosThreadRwlockUnlock(&pMgmt->lock); + return diskId; +} + int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { STfs *pTfs = pMgmt->pTfs; int32_t diskId = 0; @@ -79,7 +92,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { taosThreadRwlockRdlock(&pMgmt->lock); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL || pVnode->dropped) { + if (pVnode == NULL || pVnode->dropped || pVnode->failed) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; pVnode = NULL; } else { @@ -100,6 +113,15 @@ void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { taosThreadRwlockUnlock(&pMgmt->lock); } +static void vmFreeVnodeObj(SVnodeObj **ppVnode) { + if (!ppVnode || !(*ppVnode)) return; + + SVnodeObj *pVnode = *ppVnode; + taosMemoryFree(pVnode->path); + taosMemoryFree(pVnode); + ppVnode[0] = NULL; +} + int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodeObj *pVnode = taosMemoryCalloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { @@ -134,6 +156,12 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { } taosThreadRwlockWrlock(&pMgmt->lock); + SVnodeObj *pOld = NULL; + taosHashGetDup(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), (void *)&pOld); + if (pOld) { + ASSERT(pOld->failed); + vmFreeVnodeObj(&pOld); + } int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); taosThreadRwlockUnlock(&pMgmt->lock); @@ -223,8 +251,7 @@ _closed: vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs); } - taosMemoryFree(pVnode->path); - taosMemoryFree(pVnode); + vmFreeVnodeObj(&pVnode); } static int32_t vmRestoreVgroupId(SWrapperCfg *pCfg, STfs *pTfs) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 4b18ec4fb0..d80bc62c47 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -187,7 +187,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp pHead->vgId = ntohl(pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); - if (pVnode == NULL || pVnode->failed) { + if (pVnode == NULL) { dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); terrno = (terrno != 0) ? terrno : -1; @@ -316,7 +316,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { int32_t size = -1; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); - if (pVnode != NULL && !pVnode->failed) { + if (pVnode != NULL) { switch (qtype) { case WRITE_QUEUE: size = taosQueueItemSize(pVnode->pWriteW.queue); From 917246e2361a0eda2a4ffc33df237b297d470e75 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 23 Nov 2023 10:09:41 +0800 Subject: [PATCH 20/21] fix: allow to drop a vnode in failed mode --- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 9e3039d73f..d9c368b582 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -87,6 +87,7 @@ typedef struct { int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId); int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId); SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId); +SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict); void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d673da4be2..670eb48bd2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -749,7 +749,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); + SVnodeObj *pVnode = vmAcquireVnodeImpl(pMgmt, vgId, false); if (pVnode == NULL) { dInfo("vgId:%d, failed to drop since %s", vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 7cb31b6e5d..f5152fb943 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -87,12 +87,12 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { return diskId; } -SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { +SVnodeObj *vmAcquireVnodeImpl(SVnodeMgmt *pMgmt, int32_t vgId, bool strict) { SVnodeObj *pVnode = NULL; taosThreadRwlockRdlock(&pMgmt->lock); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); - if (pVnode == NULL || pVnode->dropped || pVnode->failed) { + if (pVnode == NULL || strict && (pVnode->dropped || pVnode->failed)) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; pVnode = NULL; } else { @@ -104,6 +104,8 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { return pVnode; } +SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { return vmAcquireVnodeImpl(pMgmt, vgId, true); } + void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { if (pVnode == NULL) return; From 8d70d023b97b4a2472fdf940e0c208a2888758c8 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 23 Nov 2023 10:12:58 +0800 Subject: [PATCH 21/21] enh: adjust a logging msg in vmRestoreVnodeInThread --- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index f5152fb943..be88e8b3fd 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -650,7 +650,7 @@ static void *vmRestoreVnodeInThread(void *param) { for (int32_t v = 0; v < pThread->vnodeNum; ++v) { SVnodeObj *pVnode = pThread->ppVnodes[v]; if (pVnode->failed) { - dError("vgId:%d, skip restoring vnode in failure mode.", pVnode->vgId); + dError("vgId:%d, cannot restore a vnode in failed mode.", pVnode->vgId); continue; }