xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/TD-30987-7

This commit is contained in:
Hongze Cheng 2024-07-19 12:51:52 +08:00
parent 3e262250d7 1a58926f14
commit d2d60990ab
92 changed files with 5431 additions and 4654 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/12-taos-sql/12-distinguished.md
View File

@ -38,7 +38,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## Windowed Queries ## Windowed Queries
Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are four kinds of windows: time window, status window, session window, and event window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows: Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are five kinds of windows: time window, status window, session window, event window, and count window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows:
```sql ```sql
window_clause: { window_clause: {

2
docs/zh/12-taos-sql/12-distinguished.md
View File

@ -39,7 +39,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## 窗口切分查询 ## 窗口切分查询
TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）四种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）、计数窗口（count window）五种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。
窗口子句语法如下： 窗口子句语法如下：

1
include/common/tmsg.h
View File

@ -3612,6 +3612,7 @@ typedef struct {
int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset);
int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset);
void tDeleteSTqOffset(void* val);
typedef struct SMqVgOffset { typedef struct SMqVgOffset {
int64_t consumerId; int64_t consumerId;

108
include/libs/stream/tstream.h
View File

@ -59,8 +59,6 @@ extern "C" {
#define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5)
#define STREAM_EXEC_T_RESUME_TASK (-6) #define STREAM_EXEC_T_RESUME_TASK (-6)
#define STREAM_EXEC_T_ADD_FAILED_TASK (-7) #define STREAM_EXEC_T_ADD_FAILED_TASK (-7)
// the load and start stream task should be executed after snode has started successfully, since the load of stream
// tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
typedef struct SStreamTask SStreamTask; typedef struct SStreamTask SStreamTask;
typedef struct SStreamQueue SStreamQueue; typedef struct SStreamQueue SStreamQueue;
@ -190,8 +188,8 @@ typedef struct {
SSDataBlock* pBlock; SSDataBlock* pBlock;
} SStreamRefDataBlock; } SStreamRefDataBlock;
SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit);
void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
typedef struct { typedef struct {
char* qmsg; char* qmsg;
@ -530,13 +528,13 @@ typedef struct STaskUpdateEntry {
typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5); SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** pTask);
void tFreeStreamTask(SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver); int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status); void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo); int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId); int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@ -631,8 +629,10 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo);
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); #if 0
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
#endif
void streamTaskInputFail(SStreamTask* pTask); void streamTaskInputFail(SStreamTask* pTask);
@ -641,8 +641,8 @@ bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId);
SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask); SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask);
const char* streamTaskGetStatusStr(ETaskStatus status); const char* streamTaskGetStatusStr(ETaskStatus status);
void streamTaskResetStatus(SStreamTask* pTask); void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask);
@ -724,43 +724,43 @@ int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* p
SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st); SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta // stream task meta
void streamMetaInit(); void streamMetaInit();
void streamMetaCleanup(); void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, int32_t vgId,
int32_t vgId, int64_t stage, startComplete_fn_t fn); int64_t stage, startComplete_fn_t fn, SStreamMeta** pMeta);
void streamMetaClose(SStreamMeta* streamMeta); void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
SStreamTask* streamMetaAcquireOneTask(SStreamTask* pTask); void streamMetaAcquireOneTask(SStreamTask* pTask);
void streamMetaClear(SStreamMeta* pMeta); void streamMetaClear(SStreamMeta* pMeta);
void streamMetaInitBackend(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta);
int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
void streamMetaNotifyClose(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta);
void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta);
bool streamMetaTaskInTimer(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta);
int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
int64_t endTs, bool ready); int64_t endTs, bool ready);
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta); int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta);
int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs); int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta);
void streamMetaWUnLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta);
void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId); void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId);
SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pTaskList);
void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader);
void streamMetaLoadAllTasks(SStreamMeta* pMeta); void streamMetaLoadAllTasks(SStreamMeta* pMeta);
int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); int32_t streamMetaStartAllTasks(SStreamMeta* pMeta);
@ -771,6 +771,8 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer // timer
tmr_h streamTimerGetInstance(); tmr_h streamTimerGetInstance();
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
@ -787,14 +789,14 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
int32_t setCode); int32_t setCode);
int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask); int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq); int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq);
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo(); int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes);
// stream task state machine, and event handling // stream task state machine, and event handling
SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); int32_t streamCreateStateMachine(SStreamTask* pTask);
void* streamDestroyStateMachine(SStreamTaskSM* pSM); void streamDestroyStateMachine(SStreamTaskSM* pSM);
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn,
void* param); void* param);
int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn,
void* param); void* param);
int32_t streamTaskRestoreStatus(SStreamTask* pTask); int32_t streamTaskRestoreStatus(SStreamTask* pTask);
@ -805,7 +807,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp); void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
#ifdef __cplusplus #ifdef __cplusplus

2
include/os/osDef.h
View File

@ -220,7 +220,7 @@ void syslog(int unused, const char *format, ...);
// Linux, length of name must <= 16 (the last '\0' included) // Linux, length of name must <= 16 (the last '\0' included)
#define setThreadName(name) \ #define setThreadName(name) \
do { \ do { \
prctl(PR_SET_NAME, (name)); \ (void)prctl(PR_SET_NAME, (name)); \
} while (0) } while (0)
#define getThreadName(name) \ #define getThreadName(name) \
do { \ do { \

1
include/os/osMemory.h
View File

@ -55,6 +55,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n))
#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) #define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n))
#define TAOS_MEMMOVE(_d, _s, _n) ((void)memmove(_d, _s, _n))
#define taosMemoryFreeClear(ptr) \ #define taosMemoryFreeClear(ptr) \
do { \ do { \

1
include/os/osString.h
View File

@ -60,6 +60,7 @@ typedef enum { M2C = 0, C2M } ConvType;
} while (0) } while (0)
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
char *tstrdup(const char *src); char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosUcs4len(TdUcs4 *ucs4);

3
include/util/thash.h
View File

@ -28,8 +28,7 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *);
#define HASH_KEY_ALREADY_EXISTS (-2) #define HASH_NODE_EXIST(code) (code == TSDB_CODE_DUP_KEY)
#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
/** /**
* murmur hash algorithm * murmur hash algorithm

1
packaging/deb/DEBIAN/postinst
View File

@ -8,6 +8,7 @@ if command -v sudo > /dev/null; then
csudo="sudo " csudo="sudo "
fi fi
${csudo}mkdir -p ${insmetaPath}
${csudo}chmod -R 744 ${insmetaPath} ${csudo}chmod -R 744 ${insmetaPath}
cd ${insmetaPath} cd ${insmetaPath}
${csudo}./post.sh ${csudo}./post.sh

1
packaging/deb/DEBIAN/prerm
View File

@ -33,6 +33,7 @@ else
${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || :

13
packaging/deb/makedeb.sh
View File

@ -76,11 +76,11 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || :
fi fi
if [ -f "%{_compiledir}/../../../explorer/target/taos-explorer.service" ]; then if [ -f "${compile_dir}/../../../explorer/target/taos-explorer.service" ]; then
cp %{_compiledir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || : cp ${compile_dir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || :
fi fi
if [ -f "%{_compiledir}/../../../explorer/server/example/explorer.toml" ]; then if [ -f "${compile_dir}/../../../explorer/server/example/explorer.toml" ]; then
cp %{_compiledir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || : cp ${compile_dir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || :
fi fi
cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin
@ -90,6 +90,11 @@ cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin
cp ${top_dir}/../enterprise/packaging/start-all.sh ${pkg_dir}${install_home_path}/bin
cp ${top_dir}/../enterprise/packaging/stop-all.sh ${pkg_dir}${install_home_path}/bin
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/start-all.sh
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/stop-all.sh
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin

4
packaging/rpm/makerpm.sh
View File

@ -70,8 +70,8 @@ else
fi fi
cd ${top_dir} cd ${top_dir}
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper -t ver-${tdengine_ver}"
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper -t ver-${tdengine_ver}`
echo "taoskeeper_binary: ${taoskeeper_binary}" echo "taoskeeper_binary: ${taoskeeper_binary}"
cd ${package_dir} cd ${package_dir}

8
packaging/rpm/tdengine.spec
View File

@ -56,6 +56,10 @@ mkdir -p %{buildroot}%{homepath}/include
#mkdir -p %{buildroot}%{homepath}/init.d #mkdir -p %{buildroot}%{homepath}/init.d
mkdir -p %{buildroot}%{homepath}/script mkdir -p %{buildroot}%{homepath}/script
if [ -f %{_compiledir}/../packaging/cfg/taosd.service ]; then
cp %{_compiledir}/../packaging/cfg/taosd.service %{buildroot}%{homepath}/cfg
fi
cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg
if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then
cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg
@ -91,6 +95,10 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/../../enterprise/packaging/start-all.sh %{buildroot}%{homepath}/bin
cp %{_compiledir}/../../enterprise/packaging/stop-all.sh %{buildroot}%{homepath}/bin
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/start-all.sh
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/stop-all.sh
if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then
cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin

63
packaging/tools/install.sh
View File

@ -156,7 +156,7 @@ done
#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh) tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh start-all.sh stop-all.sh)
if [ "${verMode}" == "cluster" ]; then if [ "${verMode}" == "cluster" ]; then
services=(${serverName} ${adapterName} ${xname} ${explorerName} ${keeperName}) services=(${serverName} ${adapterName} ${xname} ${explorerName} ${keeperName})
elif [ "${verMode}" == "edge" ]; then elif [ "${verMode}" == "edge" ]; then
@ -221,6 +221,8 @@ function install_bin() {
${csudo}cp -r ${script_dir}/bin/remove.sh ${install_main_dir}/bin ${csudo}cp -r ${script_dir}/bin/remove.sh ${install_main_dir}/bin
else else
${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin
${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/bin
${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/bin
fi fi
if [[ "${verMode}" == "cluster" && "${verType}" != "client" ]]; then if [[ "${verMode}" == "cluster" && "${verType}" != "client" ]]; then
@ -576,7 +578,9 @@ function install_taosd_config() {
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile} ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile}
${csudo}echo "monitor 1" >>${script_dir}/cfg/${configFile} ${csudo}echo "monitor 1" >>${script_dir}/cfg/${configFile}
${csudo}echo "monitorFQDN ${serverFqdn}" >>${script_dir}/cfg/${configFile} ${csudo}echo "monitorFQDN ${serverFqdn}" >>${script_dir}/cfg/${configFile}
${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile} if [ "$verMode" == "cluster" ]; then
${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile}
fi
if [ -f "${configDir}/${configFile}" ]; then if [ -f "${configDir}/${configFile}" ]; then
${csudo}cp ${fileName} ${configDir}/${configFile}.new ${csudo}cp ${fileName} ${configDir}/${configFile}.new
@ -594,6 +598,7 @@ function install_config() {
[ ! -z $1 ] && return 0 || : # only install client [ ! -z $1 ] && return 0 || : # only install client
if ((${update_flag} == 1)); then if ((${update_flag} == 1)); then
install_taosd_config
return 0 return 0
fi fi
@ -651,9 +656,7 @@ function install_data() {
function install_connector() { function install_connector() {
if [ -d "${script_dir}/connector/" ]; then if [ -d "${script_dir}/connector/" ]; then
${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector"
${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/ || echo "failed to copy start-all.sh"
${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/ || echo "failed to copy stop-all.sh"
${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md" ${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md"
fi fi
} }
@ -910,39 +913,36 @@ function updateProduct() {
echo echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}"
[ -f ${configDir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] && [ -f ${configDir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] &&
echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml" echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml"
if [ "$verMode" == "cluster" ]; then echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml"
echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml"
fi
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}"
else else
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ./${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ./${serverName}${NC}"
[ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}"
fi fi
echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}" echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
fi fi
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo echo
echo "${productName} is updated successfully!" echo "${productName} is updated successfully!"
echo echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo start all the components : ./start-all.sh${NC}"
fi
echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}"
echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}" echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}"
fi fi
else else
@ -1007,39 +1007,36 @@ function installProduct() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}"
[ -f ${configDir}/${clientName}adapter.toml ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${configDir}/${clientName}adapter.toml ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To configure ${clientName}Adapter ${NC}\t: edit ${configDir}/${clientName}adapter.toml" echo -e "${GREEN_DARK}To configure ${clientName}Adapter ${NC}\t: edit ${configDir}/${clientName}adapter.toml"
if [ "$verMode" == "cluster" ]; then echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml"
echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml"
fi
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}"
else else
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${serverName}${NC}"
[ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}"
fi fi
echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}" echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
fi fi
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo echo
echo "${productName} is installed successfully!" echo "${productName} is installed successfully!"
echo echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo start all the components : sudo ./start-all.sh${NC}"
fi
echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}"
echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}" echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}"
fi fi
echo echo

2
packaging/tools/makepkg.sh
View File

@ -232,7 +232,7 @@ fi
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" remove_temp.sh sed -i "s/PREFIX=\"taos\"/PREFIX=\"${clientName2}\"/g" remove_temp.sh
sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh mv remove_temp.sh ${install_dir}/bin/remove.sh
fi fi

127
packaging/tools/post.sh
View File

@ -104,6 +104,8 @@ fi
function log_print(){ function log_print(){
now=$(date +"%D %T") now=$(date +"%D %T")
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
${csudo} touch ${install_log_path}
echo "$now $1" >> ${install_log_path} echo "$now $1" >> ${install_log_path}
} }
@ -235,11 +237,14 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/rmtaos || : ${csudo}rm -f ${bin_link_dir}/rmtaos || :
${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/*explorer || : ${csudo}rm -f ${bin_link_dir}/*explorer || :
${csudo}rm -f ${bin_link_dir}/start-all.sh || :
${csudo}rm -f ${bin_link_dir}/stop-all.sh || :
${csudo}chmod 0555 ${bin_dir}/* ${csudo}chmod 0555 ${bin_dir}/*
@ -275,8 +280,14 @@ function install_bin() {
if [ -x ${bin_dir}/taoskeeper ]; then if [ -x ${bin_dir}/taoskeeper ]; then
${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1 ${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1
fi fi
if [ -x ${bin_dir}/*explorer ]; then if [ -x ${bin_dir}/taos-explorer ]; then
${csudo}ln -s ${bin_dir}/*explorer ${bin_link_dir}/*explorer 2>>${install_log_path} || return 1 ${csudo}ln -s ${bin_dir}/taos-explorer ${bin_link_dir}/taos-explorer 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/start-all.sh ]; then
${csudo}ln -s ${bin_dir}/start-all.sh ${bin_link_dir}/start-all.sh 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/stop-all.sh ]; then
${csudo}ln -s ${bin_dir}/stop-all.sh ${bin_link_dir}/stop-all.sh 2>>${install_log_path} || return 1
fi fi
log_print "install bin success" log_print "install bin success"
} }
@ -474,6 +485,22 @@ function install_taoskeeper_config() {
${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir} ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir}
} }
function install_taos-explorer_config() {
if [ ! -f "${cfg_install_dir}/explorer.toml" ]; then
[ ! -d ${cfg_install_dir} ] &&
${csudo}${csudo}mkdir -p ${cfg_install_dir}
[ -f ${cfg_dir}/explorer.toml ] && ${csudo}cp ${cfg_dir}/explorer.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/explorer.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/explorer.toml
fi
[ -f ${cfg_dir}/explorer.toml ] &&
${csudo}mv ${cfg_dir}/explorer.toml ${cfg_dir}/explorer.toml.new
[ -f ${cfg_install_dir}/explorer.toml ] &&
${csudo}ln -s ${cfg_install_dir}/explorer.toml ${cfg_dir}
}
function install_config() { function install_config() {
log_print "start install config from ${cfg_dir} to ${cfg_install_dir}" log_print "start install config from ${cfg_dir} to ${cfg_install_dir}"
if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then
@ -609,30 +636,12 @@ function clean_service_on_systemd() {
function install_service_on_systemd() { function install_service_on_systemd() {
clean_service_on_systemd clean_service_on_systemd
[ -f ${script_dir}/../cfg/taosd.service ] &&\
${csudo}cp ${script_dir}/../cfg/taosd.service \
${service_config_dir}/ || :
taosd_service_config="${service_config_dir}/taosd.service" ${csudo}systemctl daemon-reload
${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}"
${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}"
${csudo}bash -c "echo >> ${taosd_service_config}"
${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}"
${csudo}bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}"
${csudo}bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}"
${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}"
${csudo}bash -c "echo >> ${taosd_service_config}"
${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}"
${csudo}systemctl enable taosd ${csudo}systemctl enable taosd
} }
@ -652,6 +661,11 @@ function install_service_on_launchctl() {
${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : ${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
fi fi
if [ -f ${install_main_dir}/service/com.taosdata.taos-explorer.plist ]; then
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist > /dev/null 2>&1 || :
${csudo}cp ${install_main_dir}/service/com.taosdata.taos-explorer.plist /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || :
fi
} }
function install_taosadapter_service() { function install_taosadapter_service() {
@ -672,6 +686,15 @@ function install_taoskeeper_service() {
fi fi
} }
function install_taos-explorer_service() {
if ((${service_mod}==0)); then
[ -f ${script_dir}/../cfg/taos-explorer.service ] &&\
${csudo}cp ${script_dir}/../cfg/taos-explorer.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
}
function install_service() { function install_service() {
log_print "start install service" log_print "start install service"
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
@ -748,8 +771,10 @@ function install_TDengine() {
install_config install_config
install_taosadapter_config install_taosadapter_config
install_taoskeeper_config install_taoskeeper_config
install_taos-explorer_config
install_taosadapter_service install_taosadapter_service
install_taoskeeper_service install_taoskeeper_service
install_taos-explorer_service
install_service install_service
install_app install_app
@ -757,40 +782,38 @@ function install_TDengine() {
#echo #echo
#echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo echo
echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" echo -e "${GREEN_DARK}To configure TDengine ${NC}\t: edit /etc/taos/taos.cfg"
echo -e "${GREEN_DARK}To configure taosAdapter ${NC}\t: edit /etc/taos/taosadapter.toml"
echo -e "${GREEN_DARK}To configure taos-explorer ${NC}\t: edit /etc/taos/explorer.toml"
if ((${service_mod}==0)); then if ((${service_mod}==0)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}systemctl start taosd${NC}" echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}systemctl start taosd${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}systemctl start taosadapter${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}systemctl start taoskeeper${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}systemctl start taos-explorer${NC}"
elif ((${service_mod}==1)); then elif ((${service_mod}==1)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}" echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd start ${RED} after${NC}" echo -e " : ${csudo}service taosd start ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}update-rc.d taosadapter default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taosadapter ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}update-rc.d taoskeeper default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taoskeeper ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}update-rc.d taos-explorer default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taos-explorer ${RED} after${NC}"
else else
echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}" echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ./taosd${NC}"
fi echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ./taosadapter${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ./taoskeeper${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ./taos-explorer${NC}"
if [ ! -z "$firstEp" ]; then
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
echo
fi fi
log_print "install TDengine successfully!" log_print "install TDengine successfully!"
echo echo
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo "TDengine is installed successfully!"
echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : taos -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
} }

359
packaging/tools/remove.sh
View File

@ -31,25 +31,22 @@ else
lib64_link_dir="/usr/local/lib" lib64_link_dir="/usr/local/lib"
inc_link_dir="/usr/local/include" inc_link_dir="/usr/local/include"
fi fi
serverName="taosd"
clientName="taos" PREFIX="taos"
uninstallScript="rmtaos" serverName="${PREFIX}d"
clientName="${PREFIX}"
uninstallScript="rm${PREFIX}"
adapterName="${PREFIX}adapter"
demoName="${PREFIX}demo"
benchmarkName="${PREFIX}Benchmark"
dumpName="${PREFIX}dump"
keeperName="${PREFIX}keeper"
xName="${PREFIX}x"
explorerName="${PREFIX}-explorer"
tarbitratorName="tarbitratord"
productName="TDengine" productName="TDengine"
serverName2="taosd" installDir="/usr/local/${PREFIX}"
clientName2="taos"
productName2="TDengine"
adapterName2="${clientName2}adapter"
demoName2="${clientName2}demo"
benchmarkName2="${clientName2}Benchmark"
dumpName2="${clientName2}dump"
keeperName2="${clientName2}keeper"
xName2="${clientName2}x"
explorerName2="${clientName2}-explorer"
uninstallScript2="rm${clientName2}"
installDir="/usr/local/${clientName2}"
#install main path #install main path
install_main_dir=${installDir} install_main_dir=${installDir}
@ -57,15 +54,11 @@ data_link_dir=${installDir}/data
log_link_dir=${installDir}/log log_link_dir=${installDir}/log
cfg_link_dir=${installDir}/cfg cfg_link_dir=${installDir}/cfg
local_bin_link_dir="/usr/local/bin" local_bin_link_dir="/usr/local/bin"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
taos_service_name=${serverName2} config_dir="/etc/${PREFIX}"
taosadapter_service_name="${clientName2}adapter"
tarbitrator_service_name="tarbitratord"
config_dir="/etc/${clientName2}"
services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"x" ${PREFIX}"-explorer" ${PREFIX}"keeper")
tools=(${PREFIX} ${PREFIX}"Benchmark" ${PREFIX}"dump" ${PREFIX}"demo" udfd set_core.sh TDinsight.sh $uninstallScript start-all.sh stop-all.sh)
csudo="" csudo=""
if command -v sudo >/dev/null; then if command -v sudo >/dev/null; then
@ -92,215 +85,26 @@ else
service_mod=2 service_mod=2
fi fi
function kill_taosadapter() { kill_service_of() {
pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_taosd() {
pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_tarbitrator() {
pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function clean_bin() {
# Remove link
${csudo}rm -f ${bin_link_dir}/${clientName} || :
${csudo}rm -f ${bin_link_dir}/${serverName} || :
echo "${serverName} is removed successfully"
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/${adapterName2} || :
echo "${adapterName2} is removed successfully"
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
${csudo}rm -f ${bin_link_dir}/${keeperName2} || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${xName2} || :
${csudo}rm -f ${bin_link_dir}/${explorerName2} || :
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
fi
}
function clean_local_bin() {
${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${local_bin_link_dir}/${demoName2} || :
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
# Remove link
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {
# Remove link
${csudo}rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo}rm -rf ${log_link_dir} || :
}
function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
if systemctl is-active --quiet ${taos_service_name}; then
echo "${taos_service_name} is running, stopping it..."
${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${taosd_service_config}
taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service"
if systemctl is-active --quiet ${taosadapter_service_name}; then
echo "${clientName2}Adapter is running, stopping it..."
${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
[ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
if systemctl is-active --quiet ${tarbitrator_service_name}; then
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
}
function clean_service_on_sysvinit() {
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
echo "${serverName2} is running, stopping it..."
${csudo}service ${serverName} stop || :
fi
if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}service tarbitratord stop || :
fi
if ((${initd_mod} == 1)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}chkconfig --del ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}chkconfig --del tarbitratord || :
fi
elif ((${initd_mod} == 2)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}insserv -r ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}insserv -r tarbitratord || :
fi
elif ((${initd_mod} == 3)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}update-rc.d -f ${serverName} remove || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}update-rc.d -f tarbitratord remove || :
fi
fi
${csudo}rm -f ${service_config_dir}/${serverName} || :
${csudo}rm -f ${service_config_dir}/tarbitratord || :
if $(which init &>/dev/null); then
${csudo}init q || :
fi
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
}
function clean_service() {
if ((${service_mod} == 0)); then
clean_service_on_systemd
elif ((${service_mod} == 1)); then
clean_service_on_sysvinit
else
if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
fi
kill_taosadapter
kill_taosd
kill_tarbitrator
fi
}
function remove_data_and_config() {
data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$data_dir" == X"" ]; then
data_dir="/var/lib/taos"
fi
log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$log_dir" == X"" ]; then
log_dir="/var/log/taos"
fi
[ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}/*
[ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}/*
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}/*
}
_kill_service_of() {
_service=$1 _service=$1
pid=$(ps -ef | grep "$_service" | grep -v "grep" | awk '{print $2}') pid=$(ps -ef | grep $_service | grep -v grep | grep -v $uninstallScript | awk '{print $2}')
if [ -n "$pid" ]; then if [ -n "$pid" ]; then
${csudo}kill -9 $pid || : ${csudo}kill -9 $pid || :
fi fi
} }
_clean_service_on_systemd_of() { clean_service_on_systemd_of() {
_service=$1 _service=$1
_service_config="${service_config_dir}/${_service}.service" _service_config="${service_config_dir}/${_service}.service"
if systemctl is-active --quiet ${_service}; then if systemctl is-active --quiet ${_service}; then
echo "taoskeeper is running, stopping it..." echo "${_service} is running, stopping it..."
${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null ${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null
fi fi
${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null ${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${_service_config} ${csudo}rm -f ${_service_config}
} }
_clean_service_on_sysvinit_of() {
clean_service_on_sysvinit_of() {
_service=$1 _service=$1
if pidof ${_service} &>/dev/null; then if pidof ${_service} &>/dev/null; then
echo "${_service} is running, stopping it..." echo "${_service} is running, stopping it..."
@ -327,45 +131,99 @@ _clean_service_on_sysvinit_of() {
fi fi
} }
_clean_service_of() { clean_service_of() {
_service=$1 _service=$1
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
_clean_service_on_systemd_of $_service clean_service_on_systemd_of $_service
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
_clean_service_on_sysvinit_of $_service clean_service_on_sysvinit_of $_service
else else
_kill_service_of $_service kill_service_of $_service
fi fi
} }
remove_taoskeeper() { remove_service_of() {
# remove taoskeeper bin _service=$1
_clean_service_of taoskeeper clean_service_of ${_service}
[ -e "${bin_link_dir}/taoskeeper" ] && ${csudo}rm -rf ${bin_link_dir}/taoskeeper if [[ -e "${bin_link_dir}/${_service}" || -e "${installDir}/bin/${_service}" || -e "${local_bin_link_dir}/${_service}" ]]; then
[ -e "${installDir}/taoskeeper" ] && ${csudo}rm -rf ${installDir}/taoskeeper ${csudo}rm -rf ${bin_link_dir}/${_service}
[ -e "${cfg_link_dir}/metrics.toml" ] || ${csudo}rm -rf ${cfg_link_dir}/metrics.toml ${csudo}rm -rf ${installDir}/bin/${_service}
echo "taosKeeper is removed successfully!" ${csudo}rm -rf ${local_bin_link_dir}/${_service}
} echo "${_service} is removed successfully!"
function uninstall_taosx() {
if [ -f ${installDir}/uninstall.sh ]; then
cd ${installDir}
bash uninstall.sh
fi fi
} }
if [ "$verMode" == "cluster" ]; then remove_tools_of() {
uninstall_taosx _tool=$1
fi kill_service_of ${_tool}
[ -e "${bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${bin_link_dir}/${_tool} || :
[ -e "${installDir}/bin/${_tool}" ] && ${csudo}rm -rf ${installDir}/bin/${_tool} || :
[ -e "${local_bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${local_bin_link_dir}/${_tool} || :
}
remove_taoskeeper remove_bin() {
# Stop service and disable booting start. for _service in "${services[@]}"; do
clean_service remove_service_of ${_service}
# Remove binary file and links done
clean_bin
# Remove links of local bin for _tool in "${tools[@]}"; do
clean_local_bin remove_tools_of ${_tool}
# Remove header file. done
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
# Remove link
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {
# Remove link
${csudo}rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo}rm -rf ${log_link_dir} || :
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
}
function remove_data_and_config() {
data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$data_dir" == X"" ]; then
data_dir="/var/lib/${PREFIX}"
fi
log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$log_dir" == X"" ]; then
log_dir="/var/log/${PREFIX}"
fi
[ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}
[ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}
}
remove_bin
clean_header clean_header
# Remove lib file # Remove lib file
clean_lib clean_lib
@ -375,8 +233,7 @@ clean_log
clean_config clean_config
# Remove data link directory # Remove data link directory
${csudo}rm -rf ${data_link_dir} || : ${csudo}rm -rf ${data_link_dir} || :
${csudo}rm -rf ${install_main_dir} || :
${csudo}rm -rf ${install_main_dir}
if [[ -e /etc/os-release ]]; then if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
else else
@ -394,6 +251,7 @@ elif echo $osinfo | grep -qwi "centos"; then
${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || :
fi fi
if [ "$osType" = "Darwin" ]; then if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
${csudo}rm -rf /Applications/TDengine.app ${csudo}rm -rf /Applications/TDengine.app
fi fi
@ -411,6 +269,7 @@ if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then
fi fi
fi fi
command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true
echo echo
echo "${productName2} is removed successfully!" echo "${productName} is removed successfully!"
echo echo

7
packaging/tools/remove_client.sh
View File

@ -38,9 +38,10 @@ if command -v sudo > /dev/null; then
fi fi
function kill_client() { function kill_client() {
if [ -n "$(ps aux | grep -v grep | grep ${clientName})" ]; then pid=$(ps -ef | grep ${clientName2} | grep -v grep | grep -v $uninstallScript2 | awk '{print $2}')
${csudo}kill -9 $pid || : if [ -n "$pid" ]; then
fi ${csudo}kill -9 $pid || :
fi
} }
function clean_bin() { function clean_bin() {

2
source/client/src/clientHb.c
View File

@ -744,7 +744,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
} }
if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) != 0) {
taosMemoryFree(user); taosMemoryFree(user);
code = terrno ? terrno : TSDB_CODE_APP_ERROR; code = terrno ? terrno : TSDB_CODE_APP_ERROR;
goto _return; goto _return;

2
source/client/src/clientMonitor.c
View File

@ -691,7 +691,7 @@ static void* monitorThreadFunc(void* param) {
tscDebug("monitorThreadFunc start"); tscDebug("monitorThreadFunc start");
int64_t quitTime = 0; int64_t quitTime = 0;
while (1) { while (1) {
if (atomic_load_32(&slowLogFlag) > 0 > 0) { if (atomic_load_32(&slowLogFlag) > 0) {
if (quitCnt == 0) { if (quitCnt == 0) {
monitorSendAllSlowLogAtQuit(); monitorSendAllSlowLogAtQuit();
if (quitCnt == 0) { if (quitCnt == 0) {

10
source/common/src/tmsg.c
View File

@ -9258,6 +9258,12 @@ void tOffsetDestroy(void *param) {
taosMemoryFreeClear(pVal->primaryKey.pData); taosMemoryFreeClear(pVal->primaryKey.pData);
} }
} }
void tDeleteSTqOffset(void *param) {
STqOffset *pVal = (STqOffset *)param;
tOffsetDestroy(&pVal->val);
}
int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1;
if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1;
@ -9297,12 +9303,12 @@ int32_t tEncodeSTqCheckInfo(SEncoder *pEncoder, const STqCheckInfo *pInfo) {
int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) { int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) {
if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1; if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1;
if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1; if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1;
int32_t sz; int32_t sz = 0;
if (tDecodeI32(pDecoder, &sz) < 0) return -1; if (tDecodeI32(pDecoder, &sz) < 0) return -1;
pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t)); pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t));
if (pInfo->colIdList == NULL) return -1; if (pInfo->colIdList == NULL) return -1;
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
int16_t colId; int16_t colId = 0;
if (tDecodeI16(pDecoder, &colId) < 0) return -1; if (tDecodeI16(pDecoder, &colId) < 0) return -1;
taosArrayPush(pInfo->colIdList, &colId); taosArrayPush(pInfo->colIdList, &colId);
} }

63
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -242,11 +242,11 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, SStreamTask* pTask = NULL;
pStream->conf.fillHistory, pStream->subTableWithoutMd5); int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory,
if (pTask == NULL) { pStream->subTableWithoutMd5, &pTask);
terrno = TSDB_CODE_OUT_OF_MEMORY; if (code != 0) {
return terrno; return code;
} }
mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
@ -353,18 +353,13 @@ static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFi
} }
} }
static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) { static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, SStreamTask** pTask) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = int32_t code = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pTask);
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5); return code;
if (pTask == NULL) {
return NULL;
}
return pTask;
} }
static void addNewTaskList(SStreamObj* pStream) { static void addNewTaskList(SStreamObj* pStream) {
@ -398,10 +393,10 @@ static void setHTasksId(SStreamObj* pStream) {
static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey, static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) { SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) {
SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam); SStreamTask* pTask = NULL;
if (pTask == NULL) { int32_t code = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
terrno = TSDB_CODE_OUT_OF_MEMORY; if (code != TSDB_CODE_SUCCESS) {
return terrno; return code;
} }
mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
@ -412,10 +407,9 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId); streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup); code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; return code;
return terrno;
} }
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
@ -498,28 +492,29 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) { static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam,
SStreamTask** pAggTask) {
*pAggTask = NULL;
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pAggTask = int32_t code =
tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5); *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pAggTask);
if (pAggTask == NULL) { return code;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
return pAggTask;
} }
static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup, static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup,
SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) { SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
int32_t code = 0; int32_t code = 0;
SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam); SStreamTask* pTask = NULL;
if (pTask == NULL) {
return terrno; code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
} }
if (pSnode != NULL) { if (pSnode != NULL) {
code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode); code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory); mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory);

8
source/dnode/snode/src/snode.c
View File

@ -41,7 +41,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
SCheckpointInfo *pChkInfo = &pTask->chkInfo; SCheckpointInfo *pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
char *p = streamTaskGetStatus(pTask)->name; char *p = streamTaskGetStatus(pTask).name;
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
@ -71,9 +71,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
startRsync(); startRsync();
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); int32_t code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE,
if (pSnode->pMeta == NULL) { taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta);
terrno = TSDB_CODE_OUT_OF_MEMORY; if (code != TSDB_CODE_SUCCESS) {
goto FAIL; goto FAIL;
} }

5
source/dnode/vnode/CMakeLists.txt
View File

@ -68,11 +68,8 @@ set(
"src/tq/tqOffset.c" "src/tq/tqOffset.c"
"src/tq/tqPush.c" "src/tq/tqPush.c"
"src/tq/tqSink.c" "src/tq/tqSink.c"
"src/tq/tqCommit.c"
"src/tq/tqStreamTask.c" "src/tq/tqStreamTask.c"
"src/tq/tqHandleSnapshot.c" "src/tq/tqSnapshot.c"
"src/tq/tqCheckInfoSnapshot.c"
"src/tq/tqOffsetSnapshot.c"
"src/tq/tqStreamStateSnap.c" "src/tq/tqStreamStateSnap.c"
"src/tq/tqStreamTaskSnap.c" "src/tq/tqStreamTaskSnap.c"

51
source/dnode/vnode/src/inc/tq.h
View File

@ -41,8 +41,6 @@ extern "C" {
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on // clang-format on
typedef struct STqOffsetStore STqOffsetStore;
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
// tqExec // tqExec
@ -101,10 +99,11 @@ struct STQ {
SHashObj* pPushMgr; // subKey -> STqHandle SHashObj* pPushMgr; // subKey -> STqHandle
SHashObj* pHandle; // subKey -> STqHandle SHashObj* pHandle; // subKey -> STqHandle
SHashObj* pCheckInfo; // topic -> SAlterCheckInfo SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
STqOffsetStore* pOffsetStore; SHashObj* pOffset; // subKey -> STqOffsetVal
TDB* pMetaDB; TDB* pMetaDB;
TTB* pExecStore; TTB* pExecStore;
TTB* pCheckStore; TTB* pCheckStore;
TTB* pOffsetStore;
SStreamMeta* pStreamMeta; SStreamMeta* pStreamMeta;
}; };
@ -128,29 +127,22 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
int32_t tqMetaOpen(STQ* pTq); int32_t tqMetaOpen(STQ* pTq);
int32_t tqMetaClose(STQ* pTq); int32_t tqMetaClose(STQ* pTq);
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen);
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen); int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen);
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle);
int32_t tqMetaRestoreCheckInfo(STQ* pTq); int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen);
int32_t tqMetaGetHandle(STQ* pTq, const char* key); int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen);
int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle);
void* tqMetaGetOffset(STQ* pTq, const char* subkey);
STqOffsetStore* tqOffsetOpen(STQ* pTq); int32_t tqMetaTransform(STQ* pTq);
int32_t tqMetaTransform(STQ* pTq);
void tqOffsetClose(STqOffsetStore*);
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset);
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetCommitFile(STqOffsetStore* pStore);
// tqSink // tqSink
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule); const char* pIdStr, bool newSubTableRule);
void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data);
// tqOffset // tqOffset
char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqBuildFName(char** data, const char* path, char* name);
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name);
// tq util // tq util
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
@ -166,6 +158,25 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule);
#define TQ_ERR_GO_TO_END(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
goto END; \
} \
} while (0)
#define TQ_ERR_RETURN(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
return code; \
} \
} while (0)
#define TQ_SUBSCRIBE_NAME "subscribe"
#define TQ_OFFSET_NAME "offset-ver0"
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -258,7 +258,6 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile); int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);

33
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -70,10 +70,6 @@ typedef struct STsdbSnapRAWReader STsdbSnapRAWReader;
typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter; typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter;
typedef struct STqSnapReader STqSnapReader; typedef struct STqSnapReader STqSnapReader;
typedef struct STqSnapWriter STqSnapWriter; typedef struct STqSnapWriter STqSnapWriter;
typedef struct STqOffsetReader STqOffsetReader;
typedef struct STqOffsetWriter STqOffsetWriter;
typedef struct STqCheckInfoReader STqCheckInfoReader;
typedef struct STqCheckInfoWriter STqCheckInfoWriter;
typedef struct SStreamTaskReader SStreamTaskReader; typedef struct SStreamTaskReader SStreamTaskReader;
typedef struct SStreamTaskWriter SStreamTaskWriter; typedef struct SStreamTaskWriter SStreamTaskWriter;
typedef struct SStreamStateReader SStreamStateReader; typedef struct SStreamStateReader SStreamStateReader;
@ -222,8 +218,8 @@ typedef struct SMetaInfo {
int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader); int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
// tsdb // tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); int32_t tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force);
int tsdbClose(STsdb** pTsdb); int32_t tsdbClose(STsdb** pTsdb);
int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbBegin(STsdb* pTsdb);
// int32_t tsdbPrepareCommit(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb);
// int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
@ -266,7 +262,6 @@ int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver); int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq); int32_t tqScanWal(STQ* pTq);
int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId);
// tq-mq // tq-mq
@ -349,29 +344,17 @@ int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr);
int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter); int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter);
int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback); int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback);
// STqSnapshotReader == // STqSnapshotReader ==
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader);
int32_t tqSnapReaderClose(STqSnapReader** ppReader); int32_t tqSnapReaderClose(STqSnapReader** ppReader);
int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData); int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData);
// STqSnapshotWriter ====================================== // STqSnapshotWriter ======================================
int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter); int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter);
int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback); int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback);
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
// STqCheckInfoshotReader ==
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader); int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader); int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData);
// STqCheckInfoshotWriter ======================================
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter);
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback);
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData);
// STqOffsetReader ========================================
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader);
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader);
int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData);
// STqOffsetWriter ========================================
int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter);
int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback);
int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData);
// SStreamTaskWriter ====================================== // SStreamTaskWriter ======================================
int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader); int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader);

19
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -297,8 +297,17 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG);
pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta);
tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask);
pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); int32_t code = streamCreateStateMachine(pStreamTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = streamTaskCreateActiveChkptInfo(&pStreamTask->chkInfo.pActiveInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId); pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId);
if (!pStreamState) { if (!pStreamState) {
terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
@ -414,7 +423,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
goto _err; goto _err;
} }
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != 0) {
goto _err; goto _err;
} }
@ -540,12 +549,12 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid)
taosArrayDestroy(pUidArray); taosArrayDestroy(pUidArray);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
} else { } else {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }

129
source/dnode/vnode/src/tq/tq.c
View File

@ -76,9 +76,12 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
taosInitRWLatch(&pTq->lock); taosInitRWLatch(&pTq->lock);
pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK);
pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); pTq->pCheckInfo = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo);
pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pOffset, (FDelete)tDeleteSTqOffset);
int32_t code = tqInitialize(pTq); int32_t code = tqInitialize(pTq);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqClose(pTq); tqClose(pTq);
@ -90,26 +93,19 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
int32_t tqInitialize(STQ* pTq) { int32_t tqInitialize(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
pTq->pStreamMeta =
streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback); int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1,
if (pTq->pStreamMeta == NULL) { tqStartTaskCompleteCallback, &pTq->pStreamMeta);
return -1; if (code != TSDB_CODE_SUCCESS) {
return code;
} }
streamMetaLoadAllTasks(pTq->pStreamMeta); streamMetaLoadAllTasks(pTq->pStreamMeta);
if (tqMetaTransform(pTq) < 0) { if (tqMetaOpen(pTq) < 0) {
return -1; return -1;
} }
if (tqMetaRestoreCheckInfo(pTq) < 0) {
return -1;
}
pTq->pOffsetStore = tqOffsetOpen(pTq);
if (pTq->pOffsetStore == NULL) {
return -1;
}
return 0; return 0;
} }
@ -133,10 +129,10 @@ void tqClose(STQ* pTq) {
pIter = taosHashIterate(pTq->pPushMgr, pIter); pIter = taosHashIterate(pTq->pPushMgr, pIter);
} }
tqOffsetClose(pTq->pOffsetStore);
taosHashCleanup(pTq->pHandle); taosHashCleanup(pTq->pHandle);
taosHashCleanup(pTq->pPushMgr); taosHashCleanup(pTq->pPushMgr);
taosHashCleanup(pTq->pCheckInfo); taosHashCleanup(pTq->pCheckInfo);
taosHashCleanup(pTq->pOffset);
taosMemoryFree(pTq->path); taosMemoryFree(pTq->path);
tqMetaClose(pTq); tqMetaClose(pTq);
@ -221,7 +217,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
goto end; goto end;
} }
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, pOffset->subKey);
if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) { if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) {
tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64,
vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version);
@ -229,10 +225,14 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
} }
// save the new offset value // save the new offset value
code = tqOffsetWrite(pTq->pOffsetStore, pOffset); if (taosHashPut(pTq->pOffset, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset))){
if(code != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_INVALID_MSG; return -1;
goto end; }
if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), msg, msgLen - sizeof(vgOffset.consumerId)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
return 0; return 0;
@ -360,21 +360,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
while (1) { while (1) {
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
// 1. find handle // 1. find handle
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); code = tqMetaGetHandle(pTq, req.subKey, &pHandle);
if (pHandle == NULL) { if (code != TDB_CODE_SUCCESS) {
do { tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
if (tqMetaGetHandle(pTq, req.subKey) == 0) { terrno = TSDB_CODE_INVALID_MSG;
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); taosWUnLockLatch(&pTq->lock);
if (pHandle != NULL) { return -1;
break;
}
}
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG;
taosWUnLockLatch(&pTq->lock);
code = -1;
goto END;
} while (0);
} }
// 2. check rebalance status // 2. check rebalance status
@ -443,7 +434,7 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, vgOffset.offset.subKey); STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, vgOffset.offset.subKey);
if (pSavedOffset == NULL) { if (pSavedOffset == NULL) {
terrno = TSDB_CODE_TMQ_NO_COMMITTED; terrno = TSDB_CODE_TMQ_NO_COMMITTED;
return terrno; return terrno;
@ -523,7 +514,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
if (reqOffset.type == TMQ_OFFSET__LOG) { if (reqOffset.type == TMQ_OFFSET__LOG) {
dataRsp.common.rspOffset.version = reqOffset.version; dataRsp.common.rspOffset.version = reqOffset.version;
} else if (reqOffset.type < 0) { } else if (reqOffset.type < 0) {
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); STqOffset* pOffset = (STqOffset*)(STqOffset*)tqMetaGetOffset(pTq, req.subKey);
if (pOffset != NULL) { if (pOffset != NULL) {
if (pOffset->val.type != TMQ_OFFSET__LOG) { if (pOffset->val.type != TMQ_OFFSET__LOG) {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey); tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey);
@ -590,12 +581,14 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} }
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey); if (taosHashRemove(pTq->pOffset, pReq->subKey, strlen(pReq->subKey)) != 0) {
if (code != 0) { tqError("cannot process tq delete req %s, since no such offset in hash", pReq->subKey);
tqError("cannot process tq delete req %s, since no such offset in cache", pReq->subKey); }
if (tqMetaDeleteInfo(pTq, pTq->pOffsetStore, pReq->subKey, strlen(pReq->subKey)) != 0) {
tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey);
} }
if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { if (tqMetaDeleteInfo(pTq, pTq->pExecStore, pReq->subKey, strlen(pReq->subKey)) < 0) {
tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey); tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey);
} }
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
@ -605,18 +598,16 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
STqCheckInfo info = {0}; STqCheckInfo info = {0};
SDecoder decoder; if(tqMetaDecodeCheckInfo(&info, msg, msgLen) != 0){
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
if (tDecodeSTqCheckInfo(&decoder, &info) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
tDecoderClear(&decoder);
if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) { if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSTqCheckInfo(&info);
return -1; return -1;
} }
if (tqMetaSaveCheckInfo(pTq, info.topic, msg, msgLen) < 0) { if (tqMetaSaveInfo(pTq, pTq->pCheckStore, info.topic, strlen(info.topic), msg, msgLen) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -628,7 +619,8 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
if (tqMetaDeleteCheckInfo(pTq, msg) < 0) { if (tqMetaDeleteInfo(pTq, pTq->pCheckStore, msg, strlen(msg)) < 0) {
tqError("cannot process tq delete check info req %s, since no such check info", msg);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -652,19 +644,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId); req.oldConsumerId, req.newConsumerId);
taosRLockLatch(&pTq->lock);
STqHandle* pHandle = NULL; STqHandle* pHandle = NULL;
while (1) { (void)tqMetaGetHandle(pTq, req.subKey, &pHandle); //ignore return code
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); taosRUnLockLatch(&pTq->lock);
if (pHandle) {
break;
}
taosRLockLatch(&pTq->lock);
ret = tqMetaGetHandle(pTq, req.subKey);
taosRUnLockLatch(&pTq->lock);
if (ret < 0) {
break;
}
}
if (pHandle == NULL) { if (pHandle == NULL) {
if (req.oldConsumerId != -1) { if (req.oldConsumerId != -1) {
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
@ -675,7 +658,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
goto end; goto end;
} }
STqHandle handle = {0}; STqHandle handle = {0};
ret = tqCreateHandle(pTq, &req, &handle); ret = tqMetaCreateHandle(pTq, &req, &handle);
if (ret < 0) { if (ret < 0) {
tqDestroyTqHandle(&handle); tqDestroyTqHandle(&handle);
goto end; goto end;
@ -767,7 +750,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
SCheckpointInfo* pChkInfo = &pTask->chkInfo; SCheckpointInfo* pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name; char* p = streamTaskGetStatus(pTask).name;
const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
@ -856,7 +839,8 @@ int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) {
SStreamMeta* pMeta = pStreamTask->pMeta; SStreamMeta* pMeta = pStreamTask->pMeta;
STaskId hId = pStreamTask->hTaskInfo.id; STaskId hId = pStreamTask->hTaskInfo.id;
SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqWarn("s-task:0x%x failed to acquired it to exec step 2, scan wal quit", (int32_t) hId.taskId); tqWarn("s-task:0x%x failed to acquired it to exec step 2, scan wal quit", (int32_t) hId.taskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -874,7 +858,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to acquire stream task:0x%x during scan history data, task may have been destroyed", tqError("vgId:%d failed to acquire stream task:0x%x during scan history data, task may have been destroyed",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
@ -883,7 +868,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
// do recovery step1 // do recovery step1
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
char* pStatus = streamTaskGetStatus(pTask)->name; char* pStatus = streamTaskGetStatus(pTask).name;
// avoid multi-thread exec // avoid multi-thread exec
while (1) { while (1) {
@ -939,15 +924,15 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
if (retInfo.ret == TASK_SCANHISTORY_REXEC) { if (retInfo.ret == TASK_SCANHISTORY_REXEC) {
streamExecScanHistoryInFuture(pTask, retInfo.idleTime); streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else { } else {
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
ETaskStatus s = p->state; ETaskStatus s = p.state;
if (s == TASK_STATUS__PAUSE) { if (s == TASK_STATUS__PAUSE) {
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr, tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr,
el, pTask->execInfo.step1El, status); el, pTask->execInfo.step1El, status);
} else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) { } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) {
tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr,
p->name, pTask->execInfo.step1El); p.name, pTask->execInfo.step1El);
} }
} }
@ -961,7 +946,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
ASSERT(pTask->info.fillHistory == 1); ASSERT(pTask->info.fillHistory == 1);
// 1. get the related stream task // 1. get the related stream task
SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); SStreamTask* pStreamTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask == NULL) { if (pStreamTask == NULL) {
tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
pTask->streamTaskId.taskId, pTask->id.idStr); pTask->streamTaskId.taskId, pTask->id.idStr);
@ -1121,7 +1107,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64 tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64
" transId:%d it may have been destroyed", " transId:%d it may have been destroyed",
@ -1147,7 +1134,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
// todo save the checkpoint failed info // todo save the checkpoint failed info
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (req.mndTrigger == 1) { if (req.mndTrigger == 1) {
if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) { if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {

196
source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c
View File

@ -1,196 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
#include "tdbInt.h"
#include "tq.h"
// STqCheckInfoReader ========================================
struct STqCheckInfoReader {
STQ* pTq;
int64_t sver;
int64_t ever;
TBC* pCur;
};
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader) {
int32_t code = 0;
STqCheckInfoReader* pReader = NULL;
// alloc
pReader = (STqCheckInfoReader*)taosMemoryCalloc(1, sizeof(STqCheckInfoReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
// impl
code = tdbTbcOpen(pTq->pCheckStore, &pReader->pCur, NULL);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
code = tdbTbcMoveToFirst(pReader->pCur);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
tqInfo("vgId:%d, vnode checkinfo tq reader opened", TD_VID(pTq->pVnode));
*ppReader = pReader;
return code;
_err:
tqError("vgId:%d, vnode checkinfo tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppReader = NULL;
return code;
}
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader) {
int32_t code = 0;
tdbTbcClose((*ppReader)->pCur);
taosMemoryFree(*ppReader);
*ppReader = NULL;
return code;
}
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData) {
int32_t code = 0;
void* pKey = NULL;
void* pVal = NULL;
int32_t kLen = 0;
int32_t vLen = 0;
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
goto _exit;
}
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
pHdr->type = SNAP_DATA_TQ_CHECKINFO;
pHdr->size = vLen;
memcpy(pHdr->data, pVal, vLen);
_exit:
tdbFree(pKey);
tdbFree(pVal);
tqInfo("vgId:%d, vnode check info tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
return code;
_err:
tdbFree(pKey);
tdbFree(pVal);
tqError("vgId:%d, vnode check info tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
return code;
}
// STqCheckInfoWriter ========================================
struct STqCheckInfoWriter {
STQ* pTq;
int64_t sver;
int64_t ever;
TXN* txn;
};
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter) {
int32_t code = 0;
STqCheckInfoWriter* pWriter;
// alloc
pWriter = (STqCheckInfoWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pWriter->pTq = pTq;
pWriter->sver = sver;
pWriter->ever = ever;
if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
code = -1;
taosMemoryFree(pWriter);
goto _err;
}
*ppWriter = pWriter;
return code;
_err:
tqError("vgId:%d, tq check info writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback) {
int32_t code = 0;
STqCheckInfoWriter* pWriter = *ppWriter;
STQ* pTq = pWriter->pTq;
if (rollback) {
tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
} else {
code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err;
code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err;
}
taosMemoryFree(pWriter);
*ppWriter = NULL;
return code;
_err:
tqError("vgId:%d, tq check info writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqCheckInfo info = {0};
SDecoder decoder;
SDecoder* pDecoder = &decoder;
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = tDecodeSTqCheckInfo(pDecoder, &info);
if (code) goto _err;
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
if (code) goto _err;
code = tqMetaSaveCheckInfo(pTq, info.topic, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
if (code) goto _err;
tDecoderClear(pDecoder);
return code;
_err:
tDecoderClear(pDecoder);
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}

28
source/dnode/vnode/src/tq/tqCommit.c
View File

@ -1,28 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tq.h"
int tqCommit(STQ* pTq) {
#if 0
// stream meta commit does not be aligned to the vnode commit
if (streamMetaCommit(pTq->pStreamMeta) < 0) {
tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr());
return -1;
}
#endif
return tqOffsetCommitFile(pTq->pOffsetStore);
}

537
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -75,125 +75,98 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
return 0; return 0;
} }
int32_t tqMetaOpen(STQ* pTq) { int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) { SDecoder decoder = {0};
return -1; tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqCheckInfo(&decoder, info);
if (code != 0) {
tDeleteSTqCheckInfo(info);
return TSDB_CODE_OUT_OF_MEMORY;
} }
tDecoderClear(&decoder);
if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) {
return -1;
}
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaClose(STQ* pTq) {
if (pTq->pExecStore) {
tdbTbClose(pTq->pExecStore);
}
if (pTq->pCheckStore) {
tdbTbClose(pTq->pCheckStore);
}
tdbClose(pTq->pMetaDB);
return 0;
}
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) {
return -1;
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) {
tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key);
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) {
return -1;
}
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
SDecoder decoder;
int32_t code = 0;
tdbTbcMoveToFirst(pCur);
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
STqCheckInfo info;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
code = tDecodeSTqCheckInfo(&decoder, &info);
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
tDecoderClear(&decoder);
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}
END:
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
return code; return code;
} }
int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){
SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqOffset(&decoder, info);
if (code != 0) {
tDeleteSTqOffset(info);
return TSDB_CODE_OUT_OF_MEMORY;
}
tDecoderClear(&decoder);
return code;
}
int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_RETURN(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
}
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_RETURN(tdbTbDelete(ttb, key, kLen, txn));
TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
}
static int32_t tqMetaTransformOffsetInfo(STQ* pTq, char* path) {
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqOffsetRestoreFromFile(pTq, path));
END:
return code;
}
void* tqMetaGetOffset(STQ* pTq, const char* subkey){
void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey));
if (data == NULL) {
int vLen = 0;
if (tdbTbGet(pTq->pOffsetStore, subkey, strlen(subkey), &data, &vLen) < 0) {
tdbFree(data);
return NULL;
}
STqOffset offset = {0};
if (tqMetaDecodeOffsetInfo(&offset, data, vLen) != TDB_CODE_SUCCESS) {
tdbFree(data);
return NULL;
}
if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0){
tDeleteSTqOffset(&offset);
tdbFree(data);
return NULL;
}
tdbFree(data);
return taosHashGet(pTq->pOffset, subkey, strlen(subkey));
} else {
return data;
}
}
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
int32_t code; int32_t code = TDB_CODE_SUCCESS;
int32_t vlen; int32_t vlen;
void* buf = NULL; void* buf = NULL;
SEncoder encoder; SEncoder encoder;
tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
if (code < 0) { if (code < 0) {
goto end; goto END;
} }
tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 " epoch:%d vgId:%d", pHandle->subKey, tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 " epoch:%d vgId:%d", pHandle->subKey,
@ -202,71 +175,33 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
buf = taosMemoryCalloc(1, vlen); buf = taosMemoryCalloc(1, vlen);
if (buf == NULL) { if (buf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto END;
} }
tEncoderInit(&encoder, buf, vlen); tEncoderInit(&encoder, buf, vlen);
code = tEncodeSTqHandle(&encoder, pHandle); code = tEncodeSTqHandle(&encoder, pHandle);
if (code < 0) { if (code < 0) {
goto end; goto END;
} }
TXN* txn = NULL; TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pExecStore, key, (int)strlen(key), buf, vlen));
code = tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
if (code < 0) {
goto end;
}
code = tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn); END:
if (code < 0) {
goto end;
}
code = tdbCommit(pTq->pMetaDB, txn);
if (code < 0) {
goto end;
}
code = tdbPostCommit(pTq->pMetaDB, txn);
end:
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosMemoryFree(buf); taosMemoryFree(buf);
return code; return code;
} }
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
TXN* txn; int32_t code = TDB_CODE_SUCCESS;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) {
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
static int buildHandle(STQ* pTq, STqHandle* handle){
SVnode* pVnode = pTq->pVnode; SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
handle->pRef = walOpenRef(pVnode->pWal); handle->pRef = walOpenRef(pVnode->pWal);
if (handle->pRef == NULL) { if (handle->pRef == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
walSetRefVer(handle->pRef, handle->snapshotVer); TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer));
SReadHandle reader = { SReadHandle reader = {
.vnode = pVnode, .vnode = pVnode,
@ -282,18 +217,18 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId);
if (handle->execHandle.task == NULL) { if (handle->execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle->subKey); tqError("cannot create exec task for %s", handle->subKey);
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
void* scanner = NULL; void* scanner = NULL;
qExtractStreamScanner(handle->execHandle.task, &scanner); (void)qExtractStreamScanner(handle->execHandle.task, &scanner);
if (scanner == NULL) { if (scanner == NULL) {
tqError("cannot extract stream scanner for %s", handle->subKey); tqError("cannot extract stream scanner for %s", handle->subKey);
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner);
if (handle->execHandle.pTqReader == NULL) { if (handle->execHandle.pTqReader == NULL) {
tqError("cannot extract exec reader for %s", handle->subKey); tqError("cannot extract exec reader for %s", handle->subKey);
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
} else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0);
@ -308,7 +243,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) {
if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) {
tqError("nodesStringToNode error in sub stable, since %s", terrstr()); tqError("nodesStringToNode error in sub stable, since %s", terrstr());
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
} }
buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType, buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType,
@ -320,7 +255,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
if(ret != TDB_CODE_SUCCESS) { if(ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId); tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId);
taosArrayDestroy(tbUidList); taosArrayDestroy(tbUidList);
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid); tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
handle->execHandle.pTqReader = tqReaderOpen(pVnode); handle->execHandle.pTqReader = tqReaderOpen(pVnode);
@ -330,24 +265,23 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
return 0; return 0;
} }
static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder; SDecoder decoder = {0};
int32_t code = 0; int32_t code = TDB_CODE_SUCCESS;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
code = tDecodeSTqHandle(&decoder, handle); TQ_ERR_GO_TO_END(tDecodeSTqHandle(&decoder, handle));
if (code) goto end; TQ_ERR_GO_TO_END(tqMetaInitHandle(pTq, handle));
code = buildHandle(pTq, handle); tqInfo("tqMetaRestoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
if (code) goto end;
tqInfo("restoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
code = taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); code = taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
end: END:
tDecoderClear(&decoder); tDecoderClear(&decoder);
return code; return code;
} }
int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
@ -367,165 +301,180 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal);
if(buildHandle(pTq, handle) < 0){ if(tqMetaInitHandle(pTq, handle) < 0){
return -1; return -1;
} }
tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer); tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
} }
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* pExecStoreNew){ static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pExecStoreOld, &pCur, NULL) < 0) {
return -1;
}
TXN* txn;
if (tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
return -1;
}
void* pKey = NULL; void* pKey = NULL;
int kLen = 0; int kLen = 0;
void* pVal = NULL; void* pVal = NULL;
int vLen = 0; int vLen = 0;
TXN* txn = NULL;
tdbTbcMoveToFirst(pCur); int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
if (tdbTbUpsert(pExecStoreNew, pKey, kLen, pVal, vLen, txn) < 0) { TQ_ERR_GO_TO_END (tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn));
tqError("transform sub info error");
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
return -1;
}
} }
TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn));
TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn));
END:
tdbFree(pKey);
tdbFree(pVal);
(void)tdbTbcClose(pCur);
return code;
}
int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) {
void* data = taosHashGet(pTq->pHandle, key, strlen(key));
if(data == NULL){
int vLen = 0;
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) {
tdbFree(data);
return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
}
STqHandle handle = {0};
if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){
tdbFree(data);
tqDestroyTqHandle(&handle);
return TSDB_CODE_OUT_OF_MEMORY;
}
tdbFree(data);
*pHandle = taosHashGet(pTq->pHandle, key, strlen(key));
if(*pHandle == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
*pHandle = data;
}
return TDB_CODE_SUCCESS;
}
int32_t tqMetaOpenTdb(STQ* pTq) {
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.offset.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pOffsetStore, 0));
END:
return code;
}
static int32_t replaceTqPath(char** path){
char* tpath = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME));
taosMemoryFree(*path);
*path = tpath;
return TDB_CODE_SUCCESS;
}
static int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
int32_t code = 0;
STqCheckInfo info = {0};
TQ_ERR_GO_TO_END(tdbTbcOpen(pTq->pCheckStore, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen));
TQ_ERR_GO_TO_END(taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)));
}
info.colIdList = NULL;
END:
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
tdbTbcClose(pCur); tdbTbcClose(pCur);
tDeleteSTqCheckInfo(&info);
return code;
}
if (tdbCommit(pMetaDB, txn) < 0) { int32_t tqMetaOpen(STQ* pTq) {
return -1; char* maindb = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){
TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{
TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
taosRemoveFile(maindb);
} }
TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq));
if (tdbPostCommit(pMetaDB, txn) < 0) { END:
return -1; taosMemoryFree(maindb);
} return code;
return 0;
} }
int32_t tqMetaTransform(STQ* pTq) { int32_t tqMetaTransform(STQ* pTq) {
int32_t len = strlen(pTq->path) + 64; int32_t code = TDB_CODE_SUCCESS;
char* maindb = taosMemoryCalloc(1, len); TDB* pMetaDB = NULL;
sprintf(maindb, "%s%s%s", pTq->path, TD_DIRSEP, TDB_MAINDB_NAME); TTB* pExecStore = NULL;
TTB* pCheckStore = NULL;
char* offsetNew = NULL;
char* offset = NULL;
TQ_ERR_GO_TO_END(tqBuildFName(&offset, pTq->path, TQ_OFFSET_NAME));
if(!taosCheckExistFile(maindb)){ TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL));
taosMemoryFree(maindb); TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0));
char* tpath = taosMemoryCalloc(1, len); TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0));
if(tpath == NULL){
return -1;
}
sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe");
taosMemoryFree(pTq->path);
pTq->path = tpath;
return tqMetaOpen(pTq);
}
int32_t code = 0; TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TDB* pMetaDB = NULL; TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
TTB* pExecStore = NULL;
TTB* pCheckStore = NULL;
char* offsetNew = NULL;
char* offset = tqOffsetBuildFName(pTq->path, 0);
if(offset == NULL){
code = -1;
goto END;
}
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore));
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
if (tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL) < 0) { TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
code = -1;
goto END;
}
if (tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0) < 0) {
code = -1;
goto END;
}
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0) < 0) {
code = -1;
goto END;
}
char* tpath = taosMemoryCalloc(1, len);
if(tpath == NULL){
code = -1;
goto END;
}
sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe");
taosMemoryFree(pTq->path);
pTq->path = tpath;
if (tqMetaOpen(pTq) < 0) {
code = -1;
goto END;
}
if( tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore) < 0){
code = -1;
goto END;
}
if(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore) < 0){
code = -1;
goto END;
}
tdbTbClose(pExecStore);
pExecStore = NULL;
tdbTbClose(pCheckStore);
pCheckStore = NULL;
tdbClose(pMetaDB);
pMetaDB = NULL;
offsetNew = tqOffsetBuildFName(pTq->path, 0);
if(offsetNew == NULL){
code = -1;
goto END;
}
if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){ if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){
tqError("copy offset file error"); tqError("copy offset file error");
code = -1;
goto END;
} }
taosRemoveFile(maindb); TQ_ERR_GO_TO_END(tqMetaTransformOffsetInfo(pTq, offsetNew));
taosRemoveFile(offset); (void)taosRemoveFile(offset);
(void)taosRemoveFile(offsetNew);
END: END:
taosMemoryFree(maindb);
taosMemoryFree(offset); taosMemoryFree(offset);
taosMemoryFree(offsetNew); taosMemoryFree(offsetNew);
tdbTbClose(pExecStore); //return 0 always, so ignore
tdbTbClose(pCheckStore); (void)tdbTbClose(pExecStore);
tdbClose(pMetaDB); (void)tdbTbClose(pCheckStore);
(void)tdbClose(pMetaDB);
return code; return code;
} }
int32_t tqMetaGetHandle(STQ* pTq, const char* key) { int32_t tqMetaClose(STQ* pTq) {
void* pVal = NULL; if (pTq->pExecStore) {
int vLen = 0; (void)tdbTbClose(pTq->pExecStore);
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &pVal, &vLen) < 0) {
return -1;
} }
STqHandle handle = {0}; if (pTq->pCheckStore) {
int code = restoreHandle(pTq, pVal, vLen, &handle); (void)tdbTbClose(pTq->pCheckStore);
if (code < 0){
tqDestroyTqHandle(&handle);
} }
tdbFree(pVal); if (pTq->pOffsetStore) {
return code; (void)tdbTbClose(pTq->pOffsetStore);
} }
(void)tdbClose(pTq->pMetaDB);
return 0;
}

204
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -16,192 +16,70 @@
#include "tq.h" #include "tq.h"
struct STqOffsetStore { int32_t tqBuildFName(char** data, const char* path, char* name) {
STQ* pTq; int32_t len = strlen(path) + strlen(name) + 2;
SHashObj* pHash; // SHashObj<subscribeKey, offset> char* fname = taosMemoryCalloc(1, len);
int8_t needCommit;
};
char* tqOffsetBuildFName(const char* path, int32_t fVer) {
int32_t len = strlen(path);
char* fname = taosMemoryCalloc(1, len + 40);
if(fname == NULL) { if(fname == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
snprintf(fname, len + 40, "%s/offset-ver%d", path, fVer); int32_t code = snprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name);
return fname; if (code < 0){
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(fname);
return code;
}
*data = fname;
return TDB_CODE_SUCCESS;
} }
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); int32_t code = TDB_CODE_SUCCESS;
void* pMemBuf = NULL;
TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
return TSDB_CODE_SUCCESS; code = TDB_CODE_SUCCESS;
goto END;
} }
int32_t vgId = TD_VID(pStore->pTq->pVnode); int64_t ret = 0;
int64_t code = 0;
int32_t size = 0; int32_t size = 0;
while (1) { while (1) {
if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if ((ret = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) {
if (code == 0) { if (ret != 0) {
break; code = TSDB_CODE_INVALID_MSG;
} else {
return -1;
} }
goto END;
} }
size = htonl(size); size = htonl(size);
void* pMemBuf = taosMemoryCalloc(1, size); pMemBuf = taosMemoryCalloc(1, size);
if (pMemBuf == NULL) { if (pMemBuf == NULL) {
tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); code = TSDB_CODE_OUT_OF_MEMORY;
terrno = TSDB_CODE_OUT_OF_MEMORY; goto END;
return -1;
} }
if ((code = taosReadFile(pFile, pMemBuf, size)) != size) { if (taosReadFile(pFile, pMemBuf, size) != size) {
taosMemoryFree(pMemBuf); terrno = TSDB_CODE_INVALID_MSG;
return -1; goto END;
} }
STqOffset offset = {0}; STqOffset offset = {0};
SDecoder decoder; TQ_ERR_GO_TO_END(tqMetaDecodeOffsetInfo(&offset, pMemBuf, size));
tDecoderInit(&decoder, pMemBuf, size); code = taosHashPut(pTq->pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset));
if (tDecodeSTqOffset(&decoder, &offset) < 0) { if (code != TDB_CODE_SUCCESS) {
taosMemoryFree(pMemBuf); tDeleteSTqOffset(&offset);
tDecoderClear(&decoder); goto END;
return code;
}
tDecoderClear(&decoder);
if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) {
return -1;
}
// todo remove this
if (offset.val.type == TMQ_OFFSET__LOG) {
STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey));
if (pHandle) {
if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) {
// tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, pHandle->subKey,
// offset.val.version);
}
}
} }
TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size));
taosMemoryFree(pMemBuf); taosMemoryFree(pMemBuf);
pMemBuf = NULL;
} }
END:
taosCloseFile(&pFile); taosCloseFile(&pFile);
return TSDB_CODE_SUCCESS; taosMemoryFree(pMemBuf);
}
return code;
STqOffsetStore* tqOffsetOpen(STQ* pTq) {
STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore));
if (pStore == NULL) {
return NULL;
}
pStore->pTq = pTq;
pStore->needCommit = 0;
pTq->pOffsetStore = pStore;
pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (pStore->pHash == NULL) {
taosMemoryFree(pStore);
return NULL;
}
taosHashSetFreeFp(pStore->pHash, tOffsetDestroy);
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
if (tqOffsetRestoreFromFile(pStore, fname) < 0) {
taosMemoryFree(fname);
taosMemoryFree(pStore);
return NULL;
}
taosMemoryFree(fname);
return pStore;
}
void tqOffsetClose(STqOffsetStore* pStore) {
if(pStore == NULL) return;
tqOffsetCommitFile(pStore);
taosHashCleanup(pStore->pHash);
taosMemoryFree(pStore);
}
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) {
return (STqOffset*)taosHashGet(pStore->pHash, subscribeKey, strlen(subscribeKey));
}
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) {
pStore->needCommit = 1;
return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
}
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) {
return taosHashRemove(pStore->pHash, subscribeKey, strlen(subscribeKey));
}
int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
if (!pStore->needCommit) {
return 0;
}
// TODO file name should be with a newer version
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
const char* err = strerror(errno);
tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pStore->pHash, pIter);
if (pIter == NULL) {
break;
}
STqOffset* pOffset = (STqOffset*)pIter;
int32_t bodyLen;
int32_t code;
tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code);
if (code < 0) {
taosHashCancelIterate(pStore->pHash, pIter);
return -1;
}
int32_t totLen = INT_BYTES + bodyLen;
void* buf = taosMemoryCalloc(1, totLen);
void* abuf = POINTER_SHIFT(buf, INT_BYTES);
*(int32_t*)buf = htonl(bodyLen);
SEncoder encoder;
tEncoderInit(&encoder, abuf, bodyLen);
tEncodeSTqOffset(&encoder, pOffset);
// write file
int64_t writeLen;
if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
taosHashCancelIterate(pStore->pHash, pIter);
taosMemoryFree(buf);
return -1;
}
taosMemoryFree(buf);
}
// close and rename file
taosCloseFile(&pFile);
pStore->needCommit = 0;
return 0;
} }

168
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -1,168 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
#include "tdbInt.h"
#include "tq.h"
// STqOffsetReader ========================================
struct STqOffsetReader {
STQ* pTq;
int64_t sver;
int64_t ever;
int8_t readEnd;
};
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader) {
STqOffsetReader* pReader = NULL;
pReader = taosMemoryCalloc(1, sizeof(STqOffsetReader));
if (pReader == NULL) {
*ppReader = NULL;
return -1;
}
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
tqInfo("vgId:%d, vnode snapshot tq offset reader opened", TD_VID(pTq->pVnode));
*ppReader = pReader;
return 0;
}
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader) {
taosMemoryFree(*ppReader);
*ppReader = NULL;
return 0;
}
int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
if (pReader->readEnd != 0) return 0;
char* fname = tqOffsetBuildFName(pReader->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
taosMemoryFree(fname);
return 0;
}
int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
taosCloseFile(&pFile);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);
SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr));
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
return terrno;
}
void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr));
int64_t contLen = taosReadFile(pFile, abuf, sz);
if (contLen != sz) {
taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1;
}
buf->size = sz;
buf->type = SNAP_DATA_TQ_OFFSET;
*ppData = (uint8_t*)buf;
pReader->readEnd = 1;
taosCloseFile(&pFile);
return 0;
}
// STqOffseWriter ========================================
struct STqOffsetWriter {
STQ* pTq;
int64_t sver;
int64_t ever;
int32_t tmpFileVer;
char* fname;
};
int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter) {
int32_t code = 0;
STqOffsetWriter* pWriter;
pWriter = (STqOffsetWriter*)taosMemoryCalloc(1, sizeof(STqOffsetWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pWriter->pTq = pTq;
pWriter->sver = sver;
pWriter->ever = ever;
*ppWriter = pWriter;
return code;
_err:
tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) {
STqOffsetWriter* pWriter = *ppWriter;
STQ* pTq = pWriter->pTq;
char* fname = tqOffsetBuildFName(pTq->path, 0);
if (rollback) {
if (taosRemoveFile(pWriter->fname) < 0) {
taosMemoryFree(fname);
return -1;
}
} else {
if (taosRenameFile(pWriter->fname, fname) < 0) {
taosMemoryFree(fname);
return -1;
}
if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) {
taosMemoryFree(fname);
return -1;
}
}
taosMemoryFree(fname);
taosMemoryFree(pWriter->fname);
taosMemoryFree(pWriter);
*ppWriter = NULL;
return 0;
}
int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData) {
STQ* pTq = pWriter->pTq;
pWriter->tmpFileVer = 1;
pWriter->fname = tqOffsetBuildFName(pTq->path, pWriter->tmpFileVer);
TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE);
SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
int64_t size = pHdr->size;
if (pFile) {
int64_t contLen = taosWriteFile(pFile, pHdr->data, size);
if (contLen != size) {
taosCloseFile(&pFile);
return -1;
}
taosCloseFile(&pFile);
} else {
return -1;
}
return 0;
}

6
source/dnode/vnode/src/tq/tqRead.c
View File

@ -341,10 +341,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
memcpy(data, pBody, len); memcpy(data, pBody, len);
SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data};
*pItem = (SStreamQueueItem*)streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT); code = streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT, (SStreamDataSubmit**)pItem);
if (*pItem == NULL) { if (code != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
terrno = code;
tqError("%s failed to create data submit for stream since out of memory", id); tqError("%s failed to create data submit for stream since out of memory", id);
return code; return code;
} }

62
source/dnode/vnode/src/tq/tqHandleSnapshot.c → source/dnode/vnode/src/tq/tqSnapshot.c
View File

@ -23,9 +23,10 @@ struct STqSnapReader {
int64_t sver; int64_t sver;
int64_t ever; int64_t ever;
TBC* pCur; TBC* pCur;
int8_t type;
}; };
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader) { int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) {
int32_t code = 0; int32_t code = 0;
STqSnapReader* pReader = NULL; STqSnapReader* pReader = NULL;
@ -38,9 +39,21 @@ int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** p
pReader->pTq = pTq; pReader->pTq = pTq;
pReader->sver = sver; pReader->sver = sver;
pReader->ever = ever; pReader->ever = ever;
pReader->type = type;
// impl // impl
code = tdbTbcOpen(pTq->pExecStore, &pReader->pCur, NULL); TTB *pTb = NULL;
if (type == SNAP_DATA_TQ_CHECKINFO) {
pTb = pTq->pCheckStore;
} else if (type == SNAP_DATA_TQ_HANDLE) {
pTb = pTq->pExecStore;
} else if (type == SNAP_DATA_TQ_OFFSET) {
pTb = pTq->pOffsetStore;
} else {
code = TSDB_CODE_INVALID_MSG;
goto _err;
}
code = tdbTbcOpen(pTb, &pReader->pCur, NULL);
if (code) { if (code) {
taosMemoryFree(pReader); taosMemoryFree(pReader);
goto _err; goto _err;
@ -91,7 +104,7 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
} }
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
pHdr->type = SNAP_DATA_TQ_HANDLE; pHdr->type = pReader->type;
pHdr->size = vLen; pHdr->size = vLen;
memcpy(pHdr->data, pVal, vLen); memcpy(pHdr->data, pVal, vLen);
@ -169,7 +182,7 @@ _err:
return code; return code;
} }
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
STQ* pTq = pWriter->pTq; STQ* pTq = pWriter->pTq;
SDecoder decoder = {0}; SDecoder decoder = {0};
@ -180,7 +193,7 @@ int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
code = tDecodeSTqHandle(pDecoder, &handle); code = tDecodeSTqHandle(pDecoder, &handle);
if (code) goto end; if (code) goto end;
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
code = tqMetaSaveHandle(pTq, handle.subKey, &handle); code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
end: end:
@ -189,3 +202,42 @@ end:
tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code); tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code);
return code; return code;
} }
int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqCheckInfo info = {0};
if(tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){
goto _err;
}
code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
tDeleteSTqCheckInfo(&info);
if (code) goto _err;
return code;
_err:
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}
int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqOffset info = {0};
if(tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){
goto _err;
}
code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
tDeleteSTqOffset(&info);
if (code) goto _err;
return code;
_err:
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}

19
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -226,15 +226,15 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
} }
// not in ready state, do not handle the data from wal // not in ready state, do not handle the data from wal
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState->name); tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState.name);
return false; return false;
} }
// fill-history task has entered into the last phase, no need to anything // fill-history task has entered into the last phase, no need to anything
if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
ASSERT(pState->state == TASK_STATUS__READY); ASSERT(pState.state == TASK_STATUS__READY);
// the maximum version of data in the WAL has reached already, the step2 is done // the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer); pTask->dataRange.range.maxVer);
@ -324,7 +324,8 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
STaskId* pTaskId = taosArrayGet(pTaskList, i); STaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
continue; continue;
} }
@ -337,7 +338,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = false; *pScanIdle = false;
// seek the stored version and extract data from WAL // seek the stored version and extract data from WAL
int32_t code = setWalReaderStartOffset(pTask, vgId); code = setWalReaderStartOffset(pTask, vgId);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
@ -348,9 +349,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState->name); tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;

2
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -81,7 +81,7 @@ static int32_t tqInitTaosxRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) {
static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, bool* pBlockReturned) { SRpcMsg* pMsg, bool* pBlockReturned) {
uint64_t consumerId = pRequest->consumerId; uint64_t consumerId = pRequest->consumerId;
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); STqOffset* pOffset = (STqOffset*)tqMetaGetOffset(pTq, pRequest->subKey);
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
*pBlockReturned = false; *pBlockReturned = false;

88
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -140,17 +140,19 @@ int32_t tqStreamTaskRestoreCheckpoint(SStreamMeta* pMeta, int64_t streamId, int3
} }
tqDebug("vgId:%d restore task:0x%" PRIx64 "-0x%x checkpointId", vgId, streamId, taskId); tqDebug("vgId:%d restore task:0x%" PRIx64 "-0x%x checkpointId", vgId, streamId, taskId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("failed to acquire task:0x%x when trying to restore checkpointId", taskId); tqError("failed to acquire task:0x%x when trying to restore checkpointId", taskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
int32_t code = streamTaskSendRestoreChkptMsg(pTask); code = streamTaskSendRestoreChkptMsg(pTask);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return code; return code;
} }
// this is to process request from transaction, always return true.
int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) { int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored) {
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
@ -230,7 +232,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList); updated = streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
// send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode // send the checkpoint-source-rsp for source task to end the checkpoint trans in mnode
streamTaskSendPreparedCheckpointsourceRsp(pTask); (void) streamTaskSendCheckpointsourceRsp(pTask);
streamTaskResetStatus(pTask); streamTaskResetStatus(pTask);
streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr); streamTaskStopMonitorCheckRsp(&pTask->taskCheckInfo, pTask->id.idStr);
@ -329,7 +331,8 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId); tqDebug("s-task:0x%x recv dispatch msg from 0x%x(vgId:%d)", req.taskId, req.upstreamTaskId, req.upstreamNodeId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask) { if (pTask) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
@ -388,7 +391,8 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:0x%x vgId:%d recv dispatch-rsp from 0x%x vgId:%d", pRsp->upstreamTaskId, pRsp->upstreamNodeId, tqDebug("s-task:0x%x vgId:%d recv dispatch-rsp from 0x%x vgId:%d", pRsp->upstreamTaskId, pRsp->upstreamNodeId,
pRsp->downstreamTaskId, pRsp->downstreamNodeId); pRsp->downstreamTaskId, pRsp->downstreamNodeId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask) { if (pTask) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -411,7 +415,8 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tDecodeStreamRetrieveReq(&decoder, &req); tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
req.dstTaskId); req.dstTaskId);
@ -419,7 +424,6 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
return -1; return -1;
} }
int32_t code = 0;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
code = streamProcessRetrieveReq(pTask, &req); code = streamProcessRetrieveReq(pTask, &req);
} else { } else {
@ -484,7 +488,8 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId);
} }
SStreamTask* pTask = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId);
} }
@ -511,7 +516,8 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
@ -598,7 +604,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks); tqDebug("vgId:%d s-task:0x%x is deployed and add into meta, numOfTasks:%d", vgId, taskId, numOfTasks);
if (restored) { if (restored) {
SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId); SStreamTask* p = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &p);
if ((p != NULL) && (p->info.fillHistory == 0)) { if ((p != NULL) && (p->info.fillHistory == 0)) {
tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
} }
@ -773,7 +780,8 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
return code; return code;
} else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while } else if (type == STREAM_EXEC_T_RESUME_TASK) { // task resume to run after idle for a while
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { if (pTask != NULL) {
char* pStatus = NULL; char* pStatus = NULL;
@ -794,7 +802,8 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
return 0; return 0;
} }
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { // even in halt status, the data in inputQ must be processed if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
char* p = NULL; char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) { if (streamTaskReadyToRun(pTask, &p)) {
@ -862,7 +871,8 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
@ -875,8 +885,8 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
streamTaskClearCheckInfo(pTask, true); streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks // clear flag set during do checkpoint, and open inputQ for all upstream tasks
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
int32_t tranId = 0; int32_t tranId = 0;
int64_t activeChkId = 0; int64_t activeChkId = 0;
streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId); streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId);
@ -885,13 +895,13 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
pTask->id.idStr, activeChkId, tranId); pTask->id.idStr, activeChkId, tranId);
streamTaskSetStatusReady(pTask); streamTaskSetStatusReady(pTask);
} else if (pState->state == TASK_STATUS__UNINIT) { } else if (pState.state == TASK_STATUS__UNINIT) {
// tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); // tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr);
// ASSERT(pTask->status.downstreamReady == 0); // ASSERT(pTask->status.downstreamReady == 0);
// tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId); // tqStreamTaskRestoreCheckpoint(pMeta, pTask->id.streamId, pTask->id.taskId);
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} else { } else {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
@ -903,7 +913,8 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
@ -925,8 +936,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state == TASK_STATUS__CK) { // recv the checkpoint-source/trigger already if (pState.state == TASK_STATUS__CK) { // recv the checkpoint-source/trigger already
int32_t transId = 0; int32_t transId = 0;
int64_t checkpointId = 0; int64_t checkpointId = 0;
@ -955,7 +966,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
} else { // upstream not recv the checkpoint-source/trigger till now } else { // upstream not recv the checkpoint-source/trigger till now
ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT); ASSERT(pState.state == TASK_STATUS__READY || pState.state == TASK_STATUS__HALT);
tqWarn( tqWarn(
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger", "upstream sending checkpoint-source/trigger",
@ -971,7 +982,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError( tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
@ -990,7 +1002,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId); pReq->taskId);
@ -1003,7 +1016,8 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pHistoryTask = NULL; SStreamTask* pHistoryTask = NULL;
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); pHistoryTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask);
if (pHistoryTask == NULL) { if (pHistoryTask == NULL) {
tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
", it may have been dropped already", ", it may have been dropped already",
@ -1033,7 +1047,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
} }
streamTaskResume(pTask); streamTaskResume(pTask);
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
int32_t level = pTask->info.taskLevel; int32_t level = pTask->info.taskLevel;
if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) { if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
@ -1070,28 +1084,30 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState->name); tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
int32_t code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
if (code != 0) { if (code != 0) {
return code; return code;
} }
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask) { if (pHTask) {
taosThreadMutexLock(&pHTask->lock); taosThreadMutexLock(&pHTask->lock);
SStreamTaskState* p = streamTaskGetStatus(pHTask); SStreamTaskState p = streamTaskGetStatus(pHTask);
tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p->name); tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name);
taosThreadMutexUnlock(&pHTask->lock); taosThreadMutexUnlock(&pHTask->lock);
code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode);
@ -1121,7 +1137,8 @@ int32_t tqStreamProcessConsensusChkptRsp2(SStreamMeta* pMeta, SRpcMsg* pMsg) { r
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId); pRsp->downstreamNodeId, pRsp->downstreamTaskId);
@ -1154,7 +1171,8 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId); pMeta->vgId, req.taskId);

11
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -41,26 +41,23 @@ int32_t tsdbBegin(STsdb *pTsdb) {
if (!pTsdb) return code; if (!pTsdb) return code;
SMemTable *pMemTable; SMemTable *pMemTable;
code = tsdbMemTableCreate(pTsdb, &pMemTable); TAOS_CHECK_GOTO(tsdbMemTableCreate(pTsdb, &pMemTable), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// lock // lock
if ((code = taosThreadMutexLock(&pTsdb->mutex))) { if ((code = taosThreadMutexLock(&pTsdb->mutex))) {
code = TAOS_SYSTEM_ERROR(code); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(code), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
pTsdb->mem = pMemTable; pTsdb->mem = pMemTable;
// unlock // unlock
if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) { if ((code = taosThreadMutexUnlock(&pTsdb->mutex))) {
code = TAOS_SYSTEM_ERROR(code); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(code), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }

205
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -92,12 +92,12 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
} }
} }
code = tsdbFSetWriterOpen(&config, &committer->writer); TAOS_CHECK_GOTO(tsdbFSetWriterOpen(&config, &committer->writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -122,32 +122,28 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
committer->ctx->tbid->uid = row->uid; committer->ctx->tbid->uid = row->uid;
if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { if (metaGetInfo(committer->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
int64_t ts = TSDBROW_TS(&row->row); int64_t ts = TSDBROW_TS(&row->row);
if (ts > committer->ctx->maxKey) { if (ts > committer->ctx->maxKey) {
code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
committer->ctx->hasTSData = true; committer->ctx->hasTSData = true;
numOfRow++; numOfRow++;
code = tsdbFSetWriteRow(committer->writer, row); TAOS_CHECK_GOTO(tsdbFSetWriteRow(committer->writer, row), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbIterMergerNext(committer->dataIterMerger), &lino, _exit);
code = tsdbIterMergerNext(committer->dataIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid, tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid,
numOfRow); numOfRow);
@ -176,8 +172,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
committer->ctx->tbid->uid = record->uid; committer->ctx->tbid->uid = record->uid;
if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
@ -192,18 +187,17 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
if (!skip) { if (!skip) {
numRecord++; numRecord++;
code = tsdbFSetWriteTombRecord(committer->writer, record); TAOS_CHECK_GOTO(tsdbFSetWriteTombRecord(committer->writer, record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = tsdbIterMergerNext(committer->tombIterMerger); TAOS_CHECK_GOTO(tsdbIterMergerNext(committer->tombIterMerger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode),
committer->ctx->info->fid, numRecord); committer->ctx->info->fid, numRecord);
@ -241,11 +235,9 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
.file = fobj->f[0], .file = fobj->f[0],
}; };
code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(committer->sttReaderArray, sttReader); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->sttReaderArray, sttReader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
STFileOp op = { STFileOp op = {
.optype = TSDB_FOP_REMOVE, .optype = TSDB_FOP_REMOVE,
@ -253,15 +245,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(committer->fopArray, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->fopArray, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
_exit: _exit:
if (code) { if (code) {
tsdbCommitCloseReader(committer); tsdbCommitCloseReader(committer);
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -295,21 +287,15 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
.numOfPKs = 0, .numOfPKs = 0,
}; };
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->dataIterArray, iter), &lino, _exit);
code = TARRAY2_APPEND(committer->dataIterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// mem tomb iter // mem tomb iter
config.type = TSDB_ITER_TYPE_MEMT_TOMB; config.type = TSDB_ITER_TYPE_MEMT_TOMB;
config.memt = committer->tsdb->imem; config.memt = committer->tsdb->imem;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->tombIterArray, iter), &lino, _exit);
code = TARRAY2_APPEND(committer->tombIterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// STT // STT
SSttFileReader *sttReader; SSttFileReader *sttReader;
@ -318,34 +304,27 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
config.type = TSDB_ITER_TYPE_STT; config.type = TSDB_ITER_TYPE_STT;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->dataIterArray, iter), &lino, _exit);
code = TARRAY2_APPEND(committer->dataIterArray, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// tomb iter // tomb iter
config.type = TSDB_ITER_TYPE_STT_TOMB; config.type = TSDB_ITER_TYPE_STT_TOMB;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(committer->tombIterArray, iter); TAOS_CHECK_GOTO(TARRAY2_APPEND(committer->tombIterArray, iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// open merger // open merger
code = tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false); TAOS_CHECK_GOTO(tsdbIterMergerOpen(committer->dataIterArray, &committer->dataIterMerger, false), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true), &lino, _exit);
code = tsdbIterMergerOpen(committer->tombIterArray, &committer->tombIterMerger, true);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbCommitCloseIter(committer); tsdbCommitCloseIter(committer);
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -361,8 +340,10 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now); committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now);
tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey, tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
&committer->ctx->maxKey); &committer->ctx->maxKey);
code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did), &lino,
_exit);
tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did);
committer->ctx->tbid->suid = 0; committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0; committer->ctx->tbid->uid = 0;
@ -371,18 +352,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
ASSERT(committer->dataIterMerger == NULL); ASSERT(committer->dataIterMerger == NULL);
ASSERT(committer->writer == NULL); ASSERT(committer->writer == NULL);
code = tsdbCommitOpenReader(committer); TAOS_CHECK_GOTO(tsdbCommitOpenReader(committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbCommitOpenIter(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitOpenWriter(committer), &lino, _exit);
code = tsdbCommitOpenIter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitOpenWriter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
__func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey, __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey,
@ -395,18 +371,14 @@ static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbCommitCloseWriter(committer); TAOS_CHECK_GOTO(tsdbCommitCloseWriter(committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbCommitCloseIter(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitCloseReader(committer), &lino, _exit);
code = tsdbCommitCloseIter(committer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitCloseReader(committer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
} }
@ -417,24 +389,15 @@ static int32_t tsdbCommitFileSet(SCommitter2 *committer) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
// fset commit start TAOS_CHECK_GOTO(tsdbCommitFileSetBegin(committer), &lino, _exit);
code = tsdbCommitFileSetBegin(committer); TAOS_CHECK_GOTO(tsdbCommitTSData(committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbCommitTombData(committer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbCommitFileSetEnd(committer), &lino, _exit);
// commit fset
code = tsdbCommitTSData(committer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitTombData(committer);
TSDB_CHECK_CODE(code, lino, _exit);
// fset commit end
code = tsdbCommitFileSetEnd(committer);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(committer->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
} }
@ -464,9 +427,6 @@ static uint32_t tFileSetCommitInfoHash(const void *arg) {
} }
static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
int32_t code = 0;
int32_t lino = 0;
if (pTsdb->commitInfo) { if (pTsdb->commitInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
@ -480,12 +440,7 @@ static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
pTsdb->commitInfo->arr = NULL; pTsdb->commitInfo->arr = NULL;
taosMemoryFreeClear(pTsdb->commitInfo); taosMemoryFreeClear(pTsdb->commitInfo);
} }
return 0;
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
} }
static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
@ -494,11 +449,10 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo)); pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo));
if (pTsdb->commitInfo == NULL) { if (pTsdb->commitInfo == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare); TAOS_CHECK_GOTO(vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *)); pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *));
if (pTsdb->commitInfo->arr == NULL) { if (pTsdb->commitInfo->arr == NULL) {
@ -508,7 +462,7 @@ static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
_exit: _exit:
if (code) { if (code) {
tsdbCommitInfoDestroy(pTsdb); tsdbCommitInfoDestroy(pTsdb);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -520,22 +474,21 @@ static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) {
SFileSetCommitInfo *tinfo; SFileSetCommitInfo *tinfo;
if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) { if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
tinfo->fid = fid; tinfo->fid = fid;
tinfo->fset = NULL; tinfo->fset = NULL;
code = vHashPut(tsdb->commitInfo->ht, tinfo); TAOS_CHECK_GOTO(vHashPut(tsdb->commitInfo->ht, tinfo), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) { if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare); taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -547,8 +500,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
STFileSet *fset = NULL; STFileSet *fset = NULL;
SRBTreeIter iter; SRBTreeIter iter;
code = tsdbCommitInfoInit(tsdb); TAOS_CHECK_GOTO(tsdbCommitInfoInit(tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// scan time-series data // scan time-series data
iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
@ -581,8 +533,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
}; };
vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
if (info == NULL) { if (info == NULL) {
code = tsdbCommitInfoAdd(tsdb, fid); TAOS_CHECK_GOTO(tsdbCommitInfoAdd(tsdb, fid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
from.key.ts = maxKey + 1; from.key.ts = maxKey + 1;
@ -640,7 +591,11 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset); tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
if (fset) { if (fset) {
tsdbTFileSetInitCopy(tsdb, fset, &info->fset); code = tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
} }
} }
@ -649,7 +604,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
_exit: _exit:
if (code) { if (code) {
tsdbCommitInfoDestroy(tsdb); tsdbCommitInfoDestroy(tsdb);
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -670,12 +625,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
committer->cid = tsdbFSAllocEid(tsdb->pFS); committer->cid = tsdbFSAllocEid(tsdb->pFS);
committer->now = taosGetTimestampSec(); committer->now = taosGetTimestampSec();
code = tsdbCommitInfoBuild(tsdb); TAOS_CHECK_GOTO(tsdbCommitInfoBuild(tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
} }
@ -687,8 +641,7 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
int32_t lino = 0; int32_t lino = 0;
if (eno == 0) { if (eno == 0) {
code = tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT); TAOS_CHECK_GOTO(tsdbFSEditBegin(committer->tsdb->pFS, committer->fopArray, TSDB_FEDIT_COMMIT), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
// TODO // TODO
ASSERT(0); ASSERT(0);
@ -705,8 +658,8 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, tsdbError("vgId:%d %s failed at %s:%d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, __FILE__,
tstrerror(code), committer->cid); lino, tstrerror(code), committer->cid);
} else { } else {
tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid); tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid);
} }
@ -740,22 +693,19 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
} else { } else {
SCommitter2 committer = {0}; SCommitter2 committer = {0};
code = tsdbOpenCommitter(tsdb, info, &committer); TAOS_CHECK_GOTO(tsdbOpenCommitter(tsdb, info, &committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
code = tsdbCommitFileSet(&committer); TAOS_CHECK_GOTO(tsdbCommitFileSet(&committer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbCloseCommitter(&committer, code); TAOS_CHECK_GOTO(tsdbCloseCommitter(&committer, code), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel); tsdbInfo("vgId:%d %s done, nRow:%" PRId64 " nDel:%" PRId64, TD_VID(tsdb->pVnode), __func__, nRow, nDel);
} }
@ -792,7 +742,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} else { } else {
tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
} }
@ -805,8 +755,7 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
if (pTsdb->imem == NULL) goto _exit; if (pTsdb->imem == NULL) goto _exit;
code = tsdbFSEditAbort(pTsdb->pFS); TAOS_CHECK_GOTO(tsdbFSEditAbort(pTsdb->pFS), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pTsdb->mutex); taosThreadMutexLock(&pTsdb->mutex);
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {

77
source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c
View File

@ -23,8 +23,7 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader)); reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader));
if (reader[0] == NULL) { if (reader[0] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
reader[0]->config[0] = config[0]; reader[0]->config[0] = config[0];
@ -32,25 +31,26 @@ int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderCon
int32_t lcn = config->file.lcn; int32_t lcn = config->file.lcn;
if (fname) { if (fname) {
if (fname) { if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} else { } else {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, &config->file, fname1); tsdbTFileName(config->tsdb, &config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn); TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, lcn), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) { int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) {
if (reader[0] == NULL) return 0; if (reader[0] == NULL) {
return 0;
}
if (reader[0]->fd) { if (reader[0]->fd) {
tsdbCloseFile(&reader[0]->fd); tsdbCloseFile(&reader[0]->fd);
@ -75,12 +75,14 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey),
&lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -91,18 +93,20 @@ int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataF
int32_t lino = 0; int32_t lino = 0;
SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter)); SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter));
if (!writer) return TSDB_CODE_OUT_OF_MEMORY; if (!writer) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
writer->config[0] = config[0]; writer->config[0] = config[0];
code = tsdbDataFileRAWWriterDoOpen(writer); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterDoOpen(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
taosMemoryFree(writer); taosMemoryFree(writer);
writer = NULL; writer = NULL;
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
} }
ppWriter[0] = writer; ppWriter[0] = writer;
return code; return code;
@ -126,21 +130,20 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil
.fid = writer->config->fid, .fid = writer->config->fid,
.nf = writer->file, .nf = writer->file,
}; };
code = TARRAY2_APPEND(opArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(opArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
if (writer->fd) { if (writer->fd) {
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&writer->fd); tsdbCloseFile(&writer->fd);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -157,12 +160,12 @@ static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
} }
tsdbTFileName(writer->config->tsdb, &writer->file, fname); tsdbTFileName(writer->config->tsdb, &writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn); TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, writer->file.lcn), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -174,30 +177,30 @@ int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) {
writer->file = writer->config->file; writer->file = writer->config->file;
writer->ctx->offset = 0; writer->ctx->offset = 0;
code = tsdbDataFileRAWWriterOpenDataFD(writer); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterOpenDataFD(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
writer->ctx->opened = true; writer->ctx->opened = true;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) { int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) {
if (writer[0] == NULL) return 0; if (writer[0] == NULL) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
if (writer[0]->ctx->opened) { if (writer[0]->ctx->opened) {
if (abort) { if (abort) {
code = tsdbDataFileRAWWriterCloseAbort(writer[0]); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterCloseAbort(writer[0]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr); TAOS_CHECK_GOTO(tsdbDataFileRAWWriterCloseCommit(writer[0], opArr), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbDataFileRAWWriterDoClose(writer[0]); tsdbDataFileRAWWriterDoClose(writer[0]);
} }
@ -206,7 +209,8 @@ int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFil
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -216,15 +220,16 @@ int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDat
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength, TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data,
encryptAlgorithm, encryptKey); pDataBlock->dataLength, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
writer->ctx->offset += pDataBlock->dataLength; writer->ctx->offset += pDataBlock->dataLength;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }

721
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

File diff suppressed because it is too large Load Diff

16
source/dnode/vnode/src/tsdb/tsdbDataFileRW.h
View File

@ -98,21 +98,21 @@ int32_t tsdbDataFileFlush(SDataFileWriter *writer);
// head // head
int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize, int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmprAlg, int64_t *fileSize,
TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range, TBrinBlkArray *brinBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize, int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer, int32_t encryptAlgorithm,
int32_t encryptAlgorithm, char* encryptKey); char *encryptKey);
// tomb // tomb
int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record); int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record);
int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize, int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range, TTombBlkArray *tombBlkArray, SBuffer *buffers, SVersionRange *range,
int32_t encryptAlgorithm, char* encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize, int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey); int32_t encryptAlgorithm, char *encryptKey);
int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
int32_t encryptAlgorithm, char* encryptKey); char *encryptKey);
// utils // utils
int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer); int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer);

12
source/dnode/vnode/src/tsdb/tsdbDef.h
View File

@ -24,7 +24,7 @@ extern "C" {
#endif #endif
#define TSDB_ERROR_LOG(vid, lino, code) \ #define TSDB_ERROR_LOG(vid, lino, code) \
tsdbError("vgId:%d %s failed at line %d since %s", vid, __func__, lino, tstrerror(code)) tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code))
typedef struct SFDataPtr { typedef struct SFDataPtr {
int64_t offset; int64_t offset;
@ -34,13 +34,13 @@ typedef struct SFDataPtr {
extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn); extern int32_t tsdbOpenFile(const char *path, STsdb *pTsdb, int32_t flag, STsdbFD **ppFD, int32_t lcn);
extern void tsdbCloseFile(STsdbFD **ppFD); extern void tsdbCloseFile(STsdbFD **ppFD);
extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, extern int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size, int32_t encryptAlgorithm,
int32_t encryptAlgorithm, char* encryptKey); char *encryptKey);
extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint, extern int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey); int32_t encryptAlgorithm, char *encryptKey);
extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint, extern int32_t tsdbReadFileToBuffer(STsdbFD *pFD, int64_t offset, int64_t size, SBuffer *buffer, int64_t szHint,
int32_t encryptAlgorithm, char* encryptKey); int32_t encryptAlgorithm, char *encryptKey);
extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char* encryptKey); extern int32_t tsdbFsyncFile(STsdbFD *pFD, int32_t encryptAlgorithm, char *encryptKey);
typedef struct SColCompressInfo SColCompressInfo; typedef struct SColCompressInfo SColCompressInfo;
struct SColCompressInfo { struct SColCompressInfo {

8
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -17,7 +17,9 @@
#include "vnd.h" #include "vnd.h"
int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) { int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) {
if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) return TSDB_CODE_OUT_OF_MEMORY; if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
lvl[0]->level = level; lvl[0]->level = level;
TARRAY2_INIT(lvl[0]->fobjArr); TARRAY2_INIT(lvl[0]->fobjArr);
return 0; return 0;
@ -81,14 +83,14 @@ static int32_t tsdbSttLvlFilteredInitEx(STsdb *pTsdb, const SSttLvl *lvl1, int64
return code; return code;
} }
TARRAY2_APPEND(lvl[0]->fobjArr, fobj); TAOS_CHECK_RETURN(TARRAY2_APPEND(lvl[0]->fobjArr, fobj));
} else { } else {
STFileOp op = { STFileOp op = {
.optype = TSDB_FOP_REMOVE, .optype = TSDB_FOP_REMOVE,
.fid = fobj1->f->fid, .fid = fobj1->f->fid,
.of = fobj1->f[0], .of = fobj1->f[0],
}; };
TARRAY2_APPEND(fopArr, op); TAOS_CHECK_RETURN(TARRAY2_APPEND(fopArr, op));
} }
} }
return 0; return 0;

92
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -93,11 +93,11 @@ static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
} }
// EXPOSED APIS ================================================== // EXPOSED APIS ==================================================
static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { static char *getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) {
const char* p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); const char *p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did);
int32_t len = strlen(p1); int32_t len = strlen(p1);
char* p = memcpy(fname, p1, len); char *p = memcpy(fname, p1, len);
p += len; p += len;
*(p++) = TD_DIRSEP[0]; *(p++) = TD_DIRSEP[0];
@ -125,99 +125,31 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t
} }
void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) { void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname);
memcpy(p, ".head", 5); memcpy(p, ".head", 5);
p[5] = 0; p[5] = 0;
} }
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) { void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname);
memcpy(p, ".data", 5); memcpy(p, ".data", 5);
p[5] = 0; p[5] = 0;
} }
void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname);
memcpy(p, ".stt", 4); memcpy(p, ".stt", 4);
p[4] = 0; p[4] = 0;
} }
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname);
memcpy(p, ".sma", 4); memcpy(p, ".sma", 4);
p[4] = 0; p[4] = 0;
} }
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; } bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2) { return pDelFile1->commitID == pDelFile2->commitID; }
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
int32_t code = 0;
int64_t size = 0;
int64_t n;
TdFilePtr pFD;
char fname[TSDB_FILENAME_LEN] = {0};
char hdr[TSDB_FHDR_SIZE] = {0};
// truncate
switch (ftype) {
case TSDB_DATA_FILE:
size = pSet->pDataF->size;
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
tPutDataFile(hdr, pSet->pDataF);
break;
case TSDB_SMA_FILE:
size = pSet->pSmaF->size;
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
tPutSmaFile(hdr, pSet->pSmaF);
break;
default:
goto _err; // make the coverity scan happy
}
taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
// open
pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_WRITE_THROUGH);
if (pFD == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// ftruncate
if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// update header
n = taosLSeekFile(pFD, 0, SEEK_SET);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
n = taosWriteFile(pFD, hdr, TSDB_FHDR_SIZE);
if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// sync
if (taosFsyncFile(pFD) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
// close
taosCloseFile(&pFD);
return code;
_err:
tsdbError("vgId:%d, tsdb rollback file failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code;
}
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0; int32_t n = 0;
@ -249,7 +181,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
// head // head
pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
if (pSet->pHeadF == NULL) { if (pSet->pHeadF == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->pHeadF->nRef = 1; pSet->pHeadF->nRef = 1;
n += tGetHeadFile(p + n, pSet->pHeadF); n += tGetHeadFile(p + n, pSet->pHeadF);
@ -257,7 +189,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
// data // data
pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile));
if (pSet->pDataF == NULL) { if (pSet->pDataF == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->pDataF->nRef = 1; pSet->pDataF->nRef = 1;
n += tGetDataFile(p + n, pSet->pDataF); n += tGetDataFile(p + n, pSet->pDataF);
@ -265,7 +197,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
// sma // sma
pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile));
if (pSet->pSmaF == NULL) { if (pSet->pSmaF == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->pSmaF->nRef = 1; pSet->pSmaF->nRef = 1;
n += tGetSmaFile(p + n, pSet->pSmaF); n += tGetSmaFile(p + n, pSet->pSmaF);
@ -275,7 +207,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile));
if (pSet->aSttF[iStt] == NULL) { if (pSet->aSttF[iStt] == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSet->aSttF[iStt]->nRef = 1; pSet->aSttF[iStt]->nRef = 1;
n += tGetSttFile(p + n, pSet->aSttF[iStt]); n += tGetSttFile(p + n, pSet->aSttF[iStt]);

20
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -165,8 +165,7 @@ static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_j
static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); } static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t stt_to_json(const STFile *file, cJSON *json) { static int32_t stt_to_json(const STFile *file, cJSON *json) {
int32_t code = tfile_to_json(file, json); TAOS_CHECK_RETURN(tfile_to_json(file, json));
if (code) return code;
/* lvl */ /* lvl */
if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) { if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) {
@ -181,8 +180,7 @@ static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_fr
static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); } static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t stt_from_json(const cJSON *json, STFile *file) { static int32_t stt_from_json(const cJSON *json, STFile *file) {
int32_t code = tfile_from_json(json, file); TAOS_CHECK_RETURN(tfile_from_json(json, file));
if (code) return code;
const cJSON *item; const cJSON *item;
@ -202,7 +200,9 @@ int32_t tsdbTFileToJson(const STFile *file, cJSON *json) {
return g_tfile_info[file->type].to_json(file, json); return g_tfile_info[file->type].to_json(file, json);
} else { } else {
cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix); cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix);
if (item == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (item == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return g_tfile_info[file->type].to_json(file, item); return g_tfile_info[file->type].to_json(file, item);
} }
} }
@ -211,13 +211,11 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
f[0] = (STFile){.type = ftype}; f[0] = (STFile){.type = ftype};
if (ftype == TSDB_FTYPE_STT) { if (ftype == TSDB_FTYPE_STT) {
int32_t code = g_tfile_info[ftype].from_json(json, f); TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f));
if (code) return code;
} else { } else {
const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix); const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix);
if (cJSON_IsObject(item)) { if (cJSON_IsObject(item)) {
int32_t code = g_tfile_info[ftype].from_json(item, f); TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f));
if (code) return code;
} else { } else {
return TSDB_CODE_NOT_FOUND; return TSDB_CODE_NOT_FOUND;
} }
@ -228,7 +226,9 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
fobj[0] = taosMemoryMalloc(sizeof(*fobj[0])); fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));
if (!fobj[0]) return TSDB_CODE_OUT_OF_MEMORY; if (!fobj[0]) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexInit(&fobj[0]->mutex, NULL); taosThreadMutexInit(&fobj[0]->mutex, NULL);
fobj[0]->f[0] = f[0]; fobj[0]->f[0] = f[0];

16
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -370,7 +370,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
if (pTbData == NULL) { if (pTbData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
pTbData->suid = suid; pTbData->suid = suid;
pTbData->uid = uid; pTbData->uid = uid;
@ -401,7 +401,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
code = tsdbMemTableRehash(pMemTable); code = tsdbMemTableRehash(pMemTable);
if (code) { if (code) {
taosWUnLockLatch(&pMemTable->latch); taosWUnLockLatch(&pMemTable->latch);
goto _err; goto _exit;
} }
} }
@ -415,11 +415,11 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
taosWUnLockLatch(&pMemTable->latch); taosWUnLockLatch(&pMemTable->latch);
_exit: _exit:
*ppTbData = pTbData; if (code) {
return code; *ppTbData = NULL;
} else {
_err: *ppTbData = pTbData;
*ppTbData = NULL; }
return code; return code;
} }
@ -798,4 +798,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
_exit: _exit:
return aTbDataP; return aTbDataP;
} }

171
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -66,7 +66,6 @@ static int32_t tsdbMergerOpen(SMerger *merger) {
} }
static int32_t tsdbMergerClose(SMerger *merger) { static int32_t tsdbMergerClose(SMerger *merger) {
int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SVnode *pVnode = merger->tsdb->pVnode; SVnode *pVnode = merger->tsdb->pVnode;
@ -82,12 +81,7 @@ static int32_t tsdbMergerClose(SMerger *merger) {
TARRAY2_DESTROY(merger->dataIterArr, NULL); TARRAY2_DESTROY(merger->dataIterArr, NULL);
TARRAY2_DESTROY(merger->sttReaderArr, NULL); TARRAY2_DESTROY(merger->sttReaderArr, NULL);
TARRAY2_DESTROY(merger->fopArr, NULL); TARRAY2_DESTROY(merger->fopArr, NULL);
return 0;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(pVnode), lino, code);
}
return code;
} }
static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) {
@ -126,8 +120,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.fid = merger->ctx->fset->fid, .fid = merger->ctx->fset->fid,
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(merger->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
SSttFileReader *reader; SSttFileReader *reader;
SSttFileReaderConfig config = { SSttFileReaderConfig config = {
@ -136,11 +129,9 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.file[0] = fobj->f[0], .file[0] = fobj->f[0],
}; };
code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &reader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(merger->sttReaderArr, reader); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->sttReaderArr, reader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
} else { } else {
@ -212,8 +203,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.fid = merger->ctx->fset->fid, .fid = merger->ctx->fset->fid,
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(merger->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
SSttFileReader *reader; SSttFileReader *reader;
SSttFileReaderConfig config = { SSttFileReaderConfig config = {
@ -222,8 +212,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
.file[0] = fobj->f[0], .file[0] = fobj->f[0],
}; };
code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); TAOS_CHECK_GOTO(tsdbSttFileReaderOpen(fobj->fname, &config, &reader), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) { if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) {
tsdbSttFileReaderClose(&reader); tsdbSttFileReaderClose(&reader);
@ -239,8 +228,9 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
tsdbMergeFileSetEndCloseReader(merger); tsdbMergeFileSetEndCloseReader(merger);
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code);
} }
return code; return code;
} }
@ -259,32 +249,25 @@ static int32_t tsdbMergeFileSetBeginOpenIter(SMerger *merger) {
config.type = TSDB_ITER_TYPE_STT; config.type = TSDB_ITER_TYPE_STT;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->dataIterArr, iter), &lino, _exit);
code = TARRAY2_APPEND(merger->dataIterArr, iter);
TSDB_CHECK_CODE(code, lino, _exit);
// tomb iter // tomb iter
config.type = TSDB_ITER_TYPE_STT_TOMB; config.type = TSDB_ITER_TYPE_STT_TOMB;
config.sttReader = sttReader; config.sttReader = sttReader;
code = tsdbIterOpen(&config, &iter); TAOS_CHECK_GOTO(tsdbIterOpen(&config, &iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(merger->tombIterArr, iter); TAOS_CHECK_GOTO(TARRAY2_APPEND(merger->tombIterArr, iter), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false); TAOS_CHECK_GOTO(tsdbIterMergerOpen(merger->dataIterArr, &merger->dataIterMerger, false), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true); TAOS_CHECK_GOTO(tsdbIterMergerOpen(merger->tombIterArr, &merger->tombIterMerger, true), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -296,10 +279,9 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
SDiskID did; SDiskID did;
int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now); int32_t level = tsdbFidLevel(merger->ctx->fset->fid, &merger->tsdb->keepCfg, merger->ctx->now);
if (tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did) < 0) {
code = TSDB_CODE_FS_NO_VALID_DISK; TAOS_CHECK_GOTO(tfsAllocDisk(merger->tsdb->pVnode->pTfs, level, &did), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did); tfsMkdirRecurAt(merger->tsdb->pVnode->pTfs, merger->tsdb->path, did);
SFSetWriterConfig config = { SFSetWriterConfig config = {
.tsdb = merger->tsdb, .tsdb = merger->tsdb,
@ -328,12 +310,11 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
} }
} }
code = tsdbFSetWriterOpen(&config, &merger->writer); TAOS_CHECK_GOTO(tsdbFSetWriterOpen(&config, &merger->writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(vid, lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", vid, __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -353,20 +334,18 @@ static int32_t tsdbMergeFileSetBegin(SMerger *merger) {
merger->ctx->tbid->uid = 0; merger->ctx->tbid->uid = 0;
// open reader // open reader
code = tsdbMergeFileSetBeginOpenReader(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenReader(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// open iterator // open iterator
code = tsdbMergeFileSetBeginOpenIter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenIter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// open writer // open writer
code = tsdbMergeFileSetBeginOpenWriter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBeginOpenWriter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -387,18 +366,14 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
code = tsdbMergeFileSetEndCloseWriter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseWriter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbMergeFileSetEndCloseIter(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseIter(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbMergeFileSetEndCloseReader(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEndCloseReader(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// edit file system // edit file system
code = tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE); TAOS_CHECK_GOTO(tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&merger->tsdb->mutex); taosThreadMutexLock(&merger->tsdb->mutex);
code = tsdbFSEditCommit(merger->tsdb->pFS); code = tsdbFSEditCommit(merger->tsdb->pFS);
@ -410,6 +385,8 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code);
} }
return code; return code;
@ -420,8 +397,7 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
int32_t lino = 0; int32_t lino = 0;
merger->ctx->fset = fset; merger->ctx->fset = fset;
code = tsdbMergeFileSetBegin(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetBegin(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// data // data
SMetaInfo info; SMetaInfo info;
@ -434,17 +410,14 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
merger->ctx->tbid->suid = row->suid; merger->ctx->tbid->suid = row->suid;
if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) { if (metaGetInfo(merger->tsdb->pVnode->pMeta, row->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(merger->dataIterMerger, merger->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
code = tsdbFSetWriteRow(merger->writer, row); TAOS_CHECK_GOTO(tsdbFSetWriteRow(merger->writer, row), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(merger->dataIterMerger); TAOS_CHECK_GOTO(tsdbIterMergerNext(merger->dataIterMerger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// tomb // tomb
@ -456,24 +429,21 @@ static int32_t tsdbMergeFileSet(SMerger *merger, STFileSet *fset) {
merger->ctx->tbid->suid = record->suid; merger->ctx->tbid->suid = record->suid;
if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { if (metaGetInfo(merger->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
code = tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid); TAOS_CHECK_GOTO(tsdbIterMergerSkipTableData(merger->tombIterMerger, merger->ctx->tbid), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} }
} }
code = tsdbFSetWriteTombRecord(merger->writer, record); TAOS_CHECK_GOTO(tsdbFSetWriteTombRecord(merger->writer, record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbIterMergerNext(merger->tombIterMerger); TAOS_CHECK_GOTO(tsdbIterMergerNext(merger->tombIterMerger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbMergeFileSetEnd(merger); TAOS_CHECK_GOTO(tsdbMergeFileSetEnd(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(merger->tsdb->pVnode), __func__, lino, tstrerror(code)); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid); tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(merger->tsdb->pVnode), __func__, fset->fid);
} }
@ -487,20 +457,18 @@ static int32_t tsdbDoMerge(SMerger *merger) {
if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0; if (TARRAY2_SIZE(merger->fset->lvlArr) == 0) return 0;
SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr); SSttLvl *lvl = TARRAY2_FIRST(merger->fset->lvlArr);
if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) return 0; if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < merger->sttTrigger) {
return 0;
}
code = tsdbMergerOpen(merger); TAOS_CHECK_GOTO(tsdbMergerOpen(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbMergeFileSet(merger, merger->fset), &lino, _exit);
TAOS_CHECK_GOTO(tsdbMergerClose(merger), &lino, _exit);
code = tsdbMergeFileSet(merger, merger->fset);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbMergerClose(merger);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(merger->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__); tsdbDebug("vgId:%d %s done", TD_VID(merger->tsdb->pVnode), __func__);
} }
@ -543,45 +511,12 @@ int32_t tsdbMerge(void *arg) {
if (merger->sttTrigger <= 1) return 0; if (merger->sttTrigger <= 1) return 0;
// copy snapshot // copy snapshot
code = tsdbMergeGetFSet(merger); TAOS_CHECK_GOTO(tsdbMergeGetFSet(merger), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (merger->fset == NULL) return 0; if (merger->fset == NULL) {
/* return 0;
bool skipMerge = false;
{
extern int8_t tsS3Enabled;
extern int32_t tsS3UploadDelaySec;
long s3Size(const char *object_name);
int32_t nlevel = tfsGetLevel(merger->tsdb->pVnode->pTfs);
if (tsS3Enabled && nlevel > 1) {
STFileObj *fobj = merger->fset->farr[TSDB_FTYPE_DATA];
if (fobj && fobj->f->did.level == nlevel - 1) {
// if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay
const char *object_name = taosDirEntryBaseName((char *)fobj->fname);
if (taosCheckExistFile(fobj->fname)) {
int32_t now = taosGetTimestampSec();
int32_t mtime = 0;
taosStatFile(fobj->fname, NULL, &mtime, NULL);
if (mtime < now - tsS3UploadDelaySec) {
skipMerge = true;
}
} else // if (s3Size(object_name) > 0)
{
skipMerge = true;
}
}
// new fset can be written with ts data
}
} }
if (skipMerge) {
code = 0;
goto _exit;
}
*/
// do merge // do merge
tsdbInfo("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); tsdbInfo("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid);
code = tsdbDoMerge(merger); code = tsdbDoMerge(merger);
@ -590,7 +525,7 @@ int32_t tsdbMerge(void *arg) {
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code); tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code);
taosMsleep(100); taosMsleep(100);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);

63
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -38,17 +38,11 @@ int64_t tsdbGetEarliestTs(STsdb *pTsdb) {
return ts; return ts;
} }
/** int32_t tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {
* @brief STsdb *pTsdb = NULL;
* int slen = 0;
* @param pVnode int32_t code;
* @param ppTsdb int32_t lino;
* @param dir
* @return int 0: success, -1: failed
*/
int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {
STsdb *pTsdb = NULL;
int slen = 0;
*ppTsdb = NULL; *ppTsdb = NULL;
slen = strlen(pVnode->path) + strlen(dir) + 2; slen = strlen(pVnode->path) + strlen(dir) + 2;
@ -56,8 +50,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
// create handle // create handle
pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
if (pTsdb == NULL) { if (pTsdb == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
pTsdb->path = (char *)&pTsdb[1]; pTsdb->path = (char *)&pTsdb[1];
@ -79,42 +72,34 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
} }
// open tsdb // open tsdb
int32_t code = tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback); TAOS_CHECK_GOTO(tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback), &lino, _exit);
if (code < 0) {
terrno = code;
goto _err;
}
if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) { if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) {
terrno = TSDB_CODE_NEED_RETRY; TAOS_CHECK_GOTO(TSDB_CODE_NEED_RETRY, &lino, _exit);
goto _err;
} }
if (tsdbOpenCache(pTsdb) < 0) { TAOS_CHECK_GOTO(tsdbOpenCache(pTsdb), &lino, _exit);
goto _err;
}
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if (tsdbOpenCompMonitor(pTsdb) < 0) { TAOS_CHECK_GOTO(tsdbOpenCompMonitor(pTsdb), &lino, _exit);
goto _err;
}
#endif #endif
tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path, _exit:
pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2, if (code) {
pTsdb->keepCfg.keepTimeOffset); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, lino, tstrerror(code));
tsdbCloseFS(&pTsdb->pFS);
*ppTsdb = pTsdb; taosThreadMutexDestroy(&pTsdb->mutex);
return 0; taosMemoryFree(pTsdb);
} else {
_err: tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path,
tsdbCloseFS(&pTsdb->pFS); pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2,
taosThreadMutexDestroy(&pTsdb->mutex); pTsdb->keepCfg.keepTimeOffset);
taosMemoryFree(pTsdb); *ppTsdb = pTsdb;
return -1; }
return code;
} }
int tsdbClose(STsdb **pTsdb) { int32_t tsdbClose(STsdb **pTsdb) {
if (*pTsdb) { if (*pTsdb) {
STsdb *pdb = *pTsdb; STsdb *pdb = *pTsdb;
tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path, tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,

67
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -48,7 +48,7 @@ typedef struct {
static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo); static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo);
static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
STsdbReader* pReader); STsdbReader* pReader);
static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRow);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
STsdbReader* pReader); STsdbReader* pReader);
static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
@ -296,8 +296,11 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo
pIter->pSttBlockReader->uid = 0; pIter->pSttBlockReader->uid = 0;
tMergeTreeClose(&pIter->pSttBlockReader->mergeTree); tMergeTreeClose(&pIter->pSttBlockReader->mergeTree);
destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);
pReader->status.pLDataIterArray = NULL;
pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES); pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
if (pReader->status.pLDataIterArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// check file the time range of coverage // check file the time range of coverage
STimeWindow win = {0}; STimeWindow win = {0};
@ -708,6 +711,10 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S
while (1) { while (1) {
int32_t code = getNextBrinRecord(&iter, &pRecord); int32_t code = getNextBrinRecord(&iter, &pRecord);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pRecord == NULL) {
break; break;
} }
@ -2025,9 +2032,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SArray* pDelList = pBlockScanInfo->delSkyline; SArray* pDelList = pBlockScanInfo->delSkyline;
int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot; int32_t pkSrcSlot = pReader->suppInfo.pkSrcSlot;
TSDBROW* pRow = NULL;
TSDBROW* piRow = NULL;
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader, &pRow);
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader, &piRow);
SRowKey* pSttKey = NULL; SRowKey* pSttKey = NULL;
if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {
@ -2201,9 +2210,10 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo,
STsdbReader* pReader) { STsdbReader* pReader) {
SRowKey rowKey = {0}; SRowKey rowKey = {0};
TSDBROW* pRow = NULL;
while (1) { while (1) {
TSDBROW* pRow = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader); getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader, &pRow);
if (!pIter->hasVal) { if (!pIter->hasVal) {
break; break;
} }
@ -2559,11 +2569,11 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
} }
if (pBlockScanInfo->iter.hasVal) { if (pBlockScanInfo->iter.hasVal) {
pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow);
} }
if (pBlockScanInfo->iiter.hasVal) { if (pBlockScanInfo->iiter.hasVal) {
piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow);
} }
// two levels of mem-table does contain the valid rows // two levels of mem-table does contain the valid rows
@ -2810,13 +2820,16 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader)
TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL};
bool hasKey = false, hasIKey = false; bool hasKey = false, hasIKey = false;
TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); TSDBROW* pRow = NULL;
TSDBROW* pIRow = NULL;
getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader, &pRow);
if (pRow != NULL) { if (pRow != NULL) {
hasKey = true; hasKey = true;
key = TSDBROW_KEY(pRow); key = TSDBROW_KEY(pRow);
} }
TSDBROW* pIRow = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader); getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader, &pIRow);
if (pIRow != NULL) { if (pIRow != NULL) {
hasIKey = true; hasIKey = true;
ikey = TSDBROW_KEY(pIRow); ikey = TSDBROW_KEY(pIRow);
@ -3742,9 +3755,11 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
return false; return false;
} }
FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) { FORCE_INLINE int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes) {
*pRes = NULL;
if (!pIter->hasVal) { if (!pIter->hasVal) {
return NULL; return TSDB_CODE_SUCCESS;
} }
int32_t order = pReader->info.order; int32_t order = pReader->info.order;
@ -3754,18 +3769,20 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
TSDBROW_INIT_KEY(pRow, key); TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) { if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false; pIter->hasVal = false;
return NULL; return TSDB_CODE_SUCCESS;
} }
// it is a valid data version // it is a valid data version
if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) {
if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
return pRow; *pRes = pRow;
return TSDB_CODE_SUCCESS;
} else { } else {
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
pReader->suppInfo.numOfPks > 0); pReader->suppInfo.numOfPks > 0);
if (!dropped) { if (!dropped) {
return pRow; *pRes = pRow;
return TSDB_CODE_SUCCESS;
} }
} }
} }
@ -3773,7 +3790,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
while (1) { while (1) {
pIter->hasVal = tsdbTbDataIterNext(pIter->iter); pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
if (!pIter->hasVal) { if (!pIter->hasVal) {
return NULL; return TSDB_CODE_SUCCESS;
} }
pRow = tsdbTbDataIterGet(pIter->iter); pRow = tsdbTbDataIterGet(pIter->iter);
@ -3781,17 +3798,19 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S
TSDBROW_INIT_KEY(pRow, key); TSDBROW_INIT_KEY(pRow, key);
if (outOfTimeWindow(key.ts, &pReader->info.window)) { if (outOfTimeWindow(key.ts, &pReader->info.window)) {
pIter->hasVal = false; pIter->hasVal = false;
return NULL; return TSDB_CODE_SUCCESS;
} }
if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) {
if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
return pRow; *pRes = pRow;
return TSDB_CODE_SUCCESS;
} else { } else {
bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
pReader->suppInfo.numOfPks > 0); pReader->suppInfo.numOfPks > 0);
if (!dropped) { if (!dropped) {
return pRow; *pRes = pRow;
return TSDB_CODE_SUCCESS;
} }
} }
} }
@ -3809,7 +3828,8 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArra
} }
// data exists but not valid // data exists but not valid
TSDBROW* pRow = getValidMemRow(pIter, pDelList, pReader); TSDBROW* pRow = NULL;
getValidMemRow(pIter, pDelList, pReader, &pRow);
if (pRow == NULL) { if (pRow == NULL) {
break; break;
} }
@ -3974,7 +3994,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
*freeTSRow = false; *freeTSRow = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { // has next point in mem/imem } else { // has next point in mem/imem
pNextRow = getValidMemRow(pIter, pDelList, pReader); getValidMemRow(pIter, pDelList, pReader, &pNextRow);
if (pNextRow == NULL) { if (pNextRow == NULL) {
*pResRow = current; *pResRow = current;
*freeTSRow = false; *freeTSRow = false;
@ -4118,8 +4138,11 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRow
static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow,
int64_t endKey, bool* freeTSRow) { int64_t endKey, bool* freeTSRow) {
TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* pRow = NULL;
TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = NULL;
getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow);
getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow);
SArray* pDelList = pBlockScanInfo->delSkyline; SArray* pDelList = pBlockScanInfo->delSkyline;
uint64_t uid = pBlockScanInfo->uid; uint64_t uid = pBlockScanInfo->uid;

161
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -27,8 +27,12 @@ static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo,
static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) { static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
int32_t num = numOfTables / pBuf->numPerBucket; int32_t num = numOfTables / pBuf->numPerBucket;
int32_t remainder = numOfTables % pBuf->numPerBucket; int32_t remainder = numOfTables % pBuf->numPerBucket;
if (pBuf->pData == NULL) { if (pBuf->pData == NULL) {
pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES); pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);
if (pBuf->pData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
@ -37,7 +41,10 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pBuf->pData, &p); void* px = taosArrayPush(pBuf->pData, &p);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (remainder > 0) { if (remainder > 0) {
@ -45,7 +52,10 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pBuf->pData, &p); void* px = taosArrayPush(pBuf->pData, &p);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
pBuf->numOfTables = numOfTables; pBuf->numOfTables = numOfTables;
@ -86,7 +96,10 @@ int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pBuf->pData, &p); void* px = taosArrayPush(pBuf->pData, &p);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (remainder > 0) { if (remainder > 0) {
@ -94,7 +107,10 @@ int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pBuf->pData, &p); void* px = taosArrayPush(pBuf->pData, &p);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
pBuf->numOfTables = numOfTables; pBuf->numOfTables = numOfTables;
@ -214,7 +230,8 @@ void clearRowKey(SRowKey* pKey) {
taosMemoryFreeClear(pKey->pks[0].pData); taosMemoryFreeClear(pKey->pks[0].pData);
} }
static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) { static int32_t initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) {
int32_t code = 0;
int32_t numOfPks = pReader->suppInfo.numOfPks; int32_t numOfPks = pReader->suppInfo.numOfPks;
bool asc = ASCENDING_TRAVERSE(pReader->info.order); bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int8_t type = pReader->suppInfo.pk.type; int8_t type = pReader->suppInfo.pk.type;
@ -225,18 +242,37 @@ static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader
int64_t skey = pReader->info.window.skey; int64_t skey = pReader->info.window.skey;
int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey; int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey;
initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);
initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc); if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} else { } else {
int64_t ekey = pReader->info.window.ekey; int64_t ekey = pReader->info.window.ekey;
int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey; int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
initRowKey(pRowKey, ts, numOfPks, type, bytes, asc); code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);
initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc); if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} }
initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc); code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc);
initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc); if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc);
return code;
} }
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
@ -248,10 +284,13 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH
pScanInfo->cleanSttBlocks = false; pScanInfo->cleanSttBlocks = false;
pScanInfo->sttBlockReturned = false; pScanInfo->sttBlockReturned = false;
initLastProcKey(pScanInfo, pReader); int32_t code = initLastProcKey(pScanInfo, pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -275,7 +314,10 @@ int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, c
} }
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
initBlockScanInfoBuf(pBuf, numOfTables); code = initBlockScanInfoBuf(pBuf, numOfTables);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) { if (pUidList->tableUidList == NULL) {
@ -419,29 +461,29 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {
pIter->blockIndex += 1; pIter->blockIndex += 1;
if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {
return TSDB_CODE_FAILED; return TSDB_CODE_SUCCESS;
} }
pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
tBrinBlockClear(&pIter->block); (void) tBrinBlockClear(&pIter->block);
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
return TSDB_CODE_FAILED; return code;
} }
pIter->recordIndex = -1; pIter->recordIndex = -1;
} }
pIter->recordIndex += 1; pIter->recordIndex += 1;
tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); int32_t code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);
*pRecord = &pIter->record; *pRecord = &pIter->record;
return TSDB_CODE_SUCCESS; return code;
} }
void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } void clearBrinBlockIter(SBrinRecordIter* pIter) { (void) tBrinBlockDestroy(&pIter->block); }
// initialize the file block access order // initialize the file block access order
// sort the file blocks according to the offset of each data block in the files // sort the file blocks according to the offset of each data block in the files
@ -607,10 +649,17 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, 0); STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, 0);
for (int32_t i = 0; i < numOfBlocks; ++i) { for (int32_t i = 0; i < numOfBlocks; ++i) {
STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};
taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); void* px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
void* p = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);
taosArrayDestroy(pTableScanInfo->pBlockList); taosArrayDestroy(pTableScanInfo->pBlockList);
pTableScanInfo->pBlockList = NULL; pTableScanInfo->pBlockList = NULL;
@ -640,11 +689,18 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
int32_t index = sup.indexPerTable[pos]++; int32_t index = sup.indexPerTable[pos]++;
SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index);
taosArrayPush(pBlockIter->blockList, pBlockInfo); void* px = taosArrayPush(pBlockIter->blockList, pBlockInfo);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo;
STableDataBlockIdx tableDataBlockIdx = {.globalIndex = numOfTotal}; STableDataBlockIdx tableDataBlockIdx = {.globalIndex = numOfTotal};
taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// set data block index overflow, in order to disable the offset comparator // set data block index overflow, in order to disable the offset comparator
if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) { if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) {
@ -752,7 +808,10 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
if (record.version <= pReader->info.verRange.maxVer) { if (record.version <= pReader->info.verRange.maxVer) {
SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
taosArrayPush(pScanInfo->pFileDelData, &delData); void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
} }
@ -806,7 +865,7 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs
ETombBlkCheckEnum ret = 0; ETombBlkCheckEnum ret = 0;
code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret); code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret);
tTombBlockDestroy(&block); (void) tTombBlockDestroy(&block);
if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) { if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) {
return code; return code;
} }
@ -858,7 +917,10 @@ int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM
p = pMemTbData->pHead; p = pMemTbData->pHead;
while (p) { while (p) {
if (p->version <= ver) { if (p->version <= ver) {
taosArrayPush(pMemDelData, p); void* px = taosArrayPush(pMemDelData, p);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
p = p->pNext; p = p->pNext;
@ -870,7 +932,10 @@ int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM
p = piMemTbData->pHead; p = piMemTbData->pHead;
while (p) { while (p) {
if (p->version <= ver) { if (p->version <= ver) {
taosArrayPush(pMemDelData, p); void* px = taosArrayPush(pMemDelData, p);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
p = p->pNext; p = p->pNext;
} }
@ -899,10 +964,13 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
SStatisBlk* p = &pStatisBlkArray->data[i]; SStatisBlk* p = &pStatisBlkArray->data[i];
STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock)); STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
tStatisBlockInit(pStatisBlock); (void) tStatisBlockInit(pStatisBlock);
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock); int32_t code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);
if (code != TSDB_CODE_SUCCESS) {
return 0;
}
double el = (taosGetTimestampMs() - st) / 1000.0; double el = (taosGetTimestampMs() - st) / 1000.0;
pBlockLoadInfo->cost.loadStatisBlocks += 1; pBlockLoadInfo->cost.loadStatisBlocks += 1;
@ -914,7 +982,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
} }
if (index >= pStatisBlock->numOfRecords) { if (index >= pStatisBlock->numOfRecords) {
tStatisBlockDestroy(pStatisBlock); (void) tStatisBlockDestroy(pStatisBlock);
taosMemoryFreeClear(pStatisBlock); taosMemoryFreeClear(pStatisBlock);
return num; return num;
} }
@ -924,7 +992,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) { while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {
p = &pStatisBlkArray->data[i]; p = &pStatisBlkArray->data[i];
if (p->minTbid.suid > suid) { if (p->minTbid.suid > suid) {
tStatisBlockDestroy(pStatisBlock); (void) tStatisBlockDestroy(pStatisBlock);
taosMemoryFreeClear(pStatisBlock); taosMemoryFreeClear(pStatisBlock);
return num; return num;
} }
@ -944,7 +1012,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
} }
} }
tStatisBlockDestroy(pStatisBlock); (void) tStatisBlockDestroy(pStatisBlock);
taosMemoryFreeClear(pStatisBlock); taosMemoryFreeClear(pStatisBlock);
return num; return num;
} }
@ -956,19 +1024,22 @@ static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlo
(*i) += 1; (*i) += 1;
(*j) = 0; (*j) = 0;
if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) { if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {
tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock); (void) tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);
} }
} }
} }
void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) { int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {
int32_t size = taosArrayGetSize(pLDIterList); int32_t size = taosArrayGetSize(pLDIterList);
if (size < numOfFileObj) { if (size < numOfFileObj) {
int32_t inc = numOfFileObj - size; int32_t inc = numOfFileObj - size;
for (int32_t k = 0; k < inc; ++k) { for (int32_t k = 0; k < inc; ++k) {
SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
taosArrayPush(pLDIterList, &pIter); void* px = taosArrayPush(pLDIterList, &pIter);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
} else if (size > numOfFileObj) { // remove unused LDataIter } else if (size > numOfFileObj) { // remove unused LDataIter
int32_t inc = size - numOfFileObj; int32_t inc = size - numOfFileObj;
@ -978,21 +1049,30 @@ void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {
destroyLDataIter(pIter); destroyLDataIter(pIter);
} }
} }
return TSDB_CODE_SUCCESS;
} }
int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) { int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {
int32_t numOfLevels = pFileSet->lvlArr->size; int32_t numOfLevels = pFileSet->lvlArr->size;
int32_t code = 0;
// add the list/iter placeholder // add the list/iter placeholder
while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) { while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) {
SArray* pList = taosArrayInit(4, POINTER_BYTES); SArray* pList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pSttFileBlockIterArray, &pList); void* px = taosArrayPush(pSttFileBlockIterArray, &pList);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
for (int32_t j = 0; j < numOfLevels; ++j) { for (int32_t j = 0; j < numOfLevels; ++j) {
SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];
SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j); SArray* pList = taosArrayGetP(pSttFileBlockIterArray, j);
doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr)); code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1010,7 +1090,10 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
} }
// add the list/iter placeholder // add the list/iter placeholder
adjustSttDataIters(pSttFileBlockIterArray, pFileSet); code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet);
if (code != TSDB_CODE_SUCCESS) {
return numOfRows;
}
for (int32_t j = 0; j < numOfLevels; ++j) { for (int32_t j = 0; j < numOfLevels; ++j) {
SSttLvl* pSttLevel = pFileSet->lvlArr->data[j]; SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];

1
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -247,7 +247,6 @@ typedef struct SDataBlockIter {
typedef struct SFileBlockDumpInfo { typedef struct SFileBlockDumpInfo {
int32_t totalRows; int32_t totalRows;
int32_t rowIndex; int32_t rowIndex;
// STsdbRowKey lastKey; // this key should be removed
bool allDumped; bool allDumped;
} SFileBlockDumpInfo; } SFileBlockDumpInfo;

248
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -38,8 +38,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(&rtner->fopArr, op); return TARRAY2_APPEND(&rtner->fopArr, op);
} }
static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) { static int32_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) {
int64_t total = 0;
int64_t interval = 1000; // 1s int64_t interval = 1000; // 1s
int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX; int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX;
int64_t offset = 0; int64_t offset = 0;
@ -49,10 +48,9 @@ static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_
int64_t n; int64_t n;
int64_t last = taosGetTimestampMs(); int64_t last = taosGetTimestampMs();
if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) { if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) {
return -1; TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
total += n;
remain -= n; remain -= n;
if (remain > 0) { if (remain > 0) {
@ -63,7 +61,7 @@ static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_
} }
} }
return total; return 0;
} }
static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
@ -77,32 +75,32 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi
tsdbTFileLastChunkName(rtner->tsdb, to, fname_to); tsdbTFileLastChunkName(rtner->tsdb, to, fname_to);
fdFrom = taosOpenFile(fname_from, TD_FILE_READ); fdFrom = taosOpenFile(fname_from, TD_FILE_READ);
if (fdFrom == NULL) code = terrno; if (fdFrom == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size); tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size);
fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno; if (fdTo == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1); int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1);
int64_t n = taosFSendFile(fdTo, fdFrom, 0, lc_size);
if (n < 0) { if (taosFSendFile(fdTo, fdFrom, 0, lc_size) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
if (fdFrom) taosCloseFile(&fdFrom); tstrerror(code));
if (fdTo) taosCloseFile(&fdTo);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
return code; return code;
} }
@ -117,30 +115,29 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile
tsdbTFileName(rtner->tsdb, to, fname); tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ); fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno; if (fdFrom == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size); tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno; if (fdTo == NULL) {
TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage), TAOS_CHECK_GOTO(tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage),
tsRetentionSpeedLimitMB); tsRetentionSpeedLimitMB),
if (n < 0) { &lino, _exit);
code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
taosCloseFile(&fdFrom); tstrerror(code));
taosCloseFile(&fdTo);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
return code; return code;
} }
@ -157,8 +154,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(&rtner->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// create new // create new
op = (STFileOp){ op = (STFileOp){
@ -181,22 +177,20 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
}, },
}; };
code = TARRAY2_APPEND(&rtner->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// do copy the file // do copy the file
if (lcn < 1) { if (lcn < 1) {
code = tsdbDoCopyFile(rtner, fobj, &op.nf); TAOS_CHECK_GOTO(tsdbDoCopyFile(rtner, fobj, &op.nf), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbDoCopyFileLC(rtner, fobj, &op.nf); TAOS_CHECK_GOTO(tsdbDoCopyFileLC(rtner, fobj, &op.nf), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -213,8 +207,7 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
int32_t lino = 0; int32_t lino = 0;
if (TARRAY2_SIZE(&rtner->fopArr) > 0) { if (TARRAY2_SIZE(&rtner->fopArr) > 0) {
code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION); TAOS_CHECK_GOTO(tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&rtner->tsdb->mutex); taosThreadMutexLock(&rtner->tsdb->mutex);
@ -231,7 +224,8 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
} }
@ -248,26 +242,19 @@ static int32_t tsdbDoRetention(SRTNer *rtner) {
if (expLevel < 0) { // remove the fileset if (expLevel < 0) { // remove the fileset
for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) { for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
if (fobj == NULL) continue; if (fobj == NULL) continue;
code = tsdbDoRemoveFileObject(rtner, fobj); TAOS_CHECK_GOTO(tsdbDoRemoveFileObject(rtner, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
SSttLvl *lvl; SSttLvl *lvl;
TARRAY2_FOREACH(fset->lvlArr, lvl) { TARRAY2_FOREACH(fset->lvlArr, lvl) {
TARRAY2_FOREACH(lvl->fobjArr, fobj) { TARRAY2_FOREACH(lvl->fobjArr, fobj) { TAOS_CHECK_GOTO(tsdbDoRemoveFileObject(rtner, fobj), &lino, _exit); }
code = tsdbDoRemoveFileObject(rtner, fobj);
TSDB_CHECK_CODE(code, lino, _exit);
}
} }
} else if (expLevel == 0) { // only migrate to upper level } else if (expLevel == 0) { // only migrate to upper level
return 0; return 0;
} else { // migrate } else { // migrate
SDiskID did; SDiskID did;
if (tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did) < 0) { TAOS_CHECK_GOTO(tfsAllocDisk(rtner->tsdb->pVnode->pTfs, expLevel, &did), &lino, _exit);
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did); tfsMkdirRecurAt(rtner->tsdb->pVnode->pTfs, rtner->tsdb->path, did);
// data // data
@ -284,25 +271,26 @@ static int32_t tsdbDoRetention(SRTNer *rtner) {
tsdbInfo("file:%s size: %" PRId64 " do migrate from %d to %d", fobj->fname, fobj->f->size, fobj->f->did.level, tsdbInfo("file:%s size: %" PRId64 " do migrate from %d to %d", fobj->fname, fobj->f->size, fobj->f->did.level,
did.level); did.level);
code = tsdbDoMigrateFileObj(rtner, fobj, &did); TAOS_CHECK_GOTO(tsdbDoMigrateFileObj(rtner, fobj, &did), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// stt // stt
SSttLvl *lvl; SSttLvl *lvl;
TARRAY2_FOREACH(fset->lvlArr, lvl) { TARRAY2_FOREACH(fset->lvlArr, lvl) {
TARRAY2_FOREACH(lvl->fobjArr, fobj) { TARRAY2_FOREACH(lvl->fobjArr, fobj) {
if (fobj->f->did.level == did.level) continue; if (fobj->f->did.level == did.level) {
continue;
}
code = tsdbDoMigrateFileObj(rtner, fobj, &did); TAOS_CHECK_GOTO(tsdbDoMigrateFileObj(rtner, fobj, &did), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
} }
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -338,15 +326,12 @@ static int32_t tsdbRetention(void *arg) {
// do retention // do retention
if (rtner.fset) { if (rtner.fset) {
if (rtnArg->s3Migrate) { if (rtnArg->s3Migrate) {
code = tsdbDoS3Migrate(&rtner); TAOS_CHECK_GOTO(tsdbDoS3Migrate(&rtner), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbDoRetention(&rtner); TAOS_CHECK_GOTO(tsdbDoRetention(&rtner), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbDoRetentionEnd(&rtner); TAOS_CHECK_GOTO(tsdbDoRetentionEnd(&rtner), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
@ -361,7 +346,7 @@ _exit:
TARRAY2_DESTROY(&rtner.fopArr, NULL); TARRAY2_DESTROY(&rtner.fopArr, NULL);
taosMemoryFree(arg); taosMemoryFree(arg);
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(pTsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -374,12 +359,11 @@ static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate)
if (!tsdb->bgTaskDisabled) { if (!tsdb->bgTaskDisabled) {
TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
code = tsdbTFileSetOpenChannel(fset); TAOS_CHECK_GOTO(tsdbTFileSetOpenChannel(fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
if (arg == NULL) { if (arg == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
} }
arg->tsdb = tsdb; arg->tsdb = tsdb;
@ -396,7 +380,7 @@ static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate)
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -446,20 +430,19 @@ static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile
tsdbTFileName(rtner->tsdb, to, fname); tsdbTFileName(rtner->tsdb, to, fname);
fdFrom = taosOpenFile(from->fname, TD_FILE_READ); fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno; if (fdFrom == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
char *object_name = taosDirEntryBaseName(fname); char *object_name = taosDirEntryBaseName(fname);
code = s3PutObjectFromFile2(from->fname, object_name, 1); TAOS_CHECK_GOTO(s3PutObjectFromFile2(from->fname, object_name, 1), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
taosCloseFile(&fdFrom);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
taosCloseFile(&fdFrom); tstrerror(code));
} }
taosCloseFile(&fdFrom);
return code; return code;
} }
@ -477,8 +460,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(&rtner->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// create new // create new
op = (STFileOp){ op = (STFileOp){
@ -501,8 +483,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
}, },
}; };
code = TARRAY2_APPEND(&rtner->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(rtner->tsdb, &op.nf, fname); tsdbTFileName(rtner->tsdb, &op.nf, fname);
@ -514,13 +495,13 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
char *dot = strrchr(object_name_prefix, '.'); char *dot = strrchr(object_name_prefix, '.');
if (!dot) { if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1; TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit);
} }
char *dot2 = strchr(object_name, '.'); char *dot2 = strchr(object_name, '.');
if (!dot) { if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1; TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit);
} }
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn); snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
@ -529,8 +510,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - fobj->f->lcn); int64_t c_offset = chunksize * (cn - fobj->f->lcn);
code = s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize); TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// copy last chunk // copy last chunk
@ -540,30 +520,30 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn); snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", fobj->f->lcn);
fdFrom = taosOpenFile(fname, TD_FILE_READ); fdFrom = taosOpenFile(fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno; if (fdFrom == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
tsdbInfo("vgId:%d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size); tsdbInfo("vgId:%d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, lc_size);
snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn); snprintf(dot2 + 1, TSDB_FQDN_LEN - (dot2 + 1 - object_name), "%d.data", lcn);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno; if (fdTo == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
if (fdFrom) taosCloseFile(&fdFrom); tstrerror(code));
if (fdTo) taosCloseFile(&fdTo);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
return code; return code;
} }
@ -581,8 +561,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
.of = fobj->f[0], .of = fobj->f[0],
}; };
code = TARRAY2_APPEND(&rtner->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// create new // create new
op = (STFileOp){ op = (STFileOp){
@ -605,8 +584,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
}, },
}; };
code = TARRAY2_APPEND(&rtner->fopArr, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(rtner->tsdb, &op.nf, fname); tsdbTFileName(rtner->tsdb, &op.nf, fname);
@ -618,7 +596,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
char *dot = strrchr(object_name_prefix, '.'); char *dot = strrchr(object_name_prefix, '.');
if (!dot) { if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1; TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit);
} }
// do copy the file // do copy the file
@ -626,8 +604,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name_prefix), "%d.data", cn);
int64_t c_offset = chunksize * (cn - 1); int64_t c_offset = chunksize * (cn - 1);
code = s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize); TAOS_CHECK_GOTO(s3PutObjectFromFileOffset(fobj->fname, object_name_prefix, c_offset, chunksize), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// copy last chunk // copy last chunk
@ -637,34 +614,34 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
dot = strchr(object_name, '.'); dot = strchr(object_name, '.');
if (!dot) { if (!dot) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino);
return -1; TAOS_CHECK_GOTO(TSDB_CODE_FAILED, &lino, _exit);
} }
snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn); snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - object_name), "%d.data", lcn);
fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ); fdFrom = taosOpenFile(fobj->fname, TD_FILE_READ);
if (fdFrom == NULL) code = terrno; if (fdFrom == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size); tsdbInfo("vgId: %d, open lcfile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, fobj->f->size);
fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
if (fdTo == NULL) code = terrno; if (fdTo == NULL) {
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
}
int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size); int64_t n = taosFSendFile(fdTo, fdFrom, &lc_offset, lc_size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at line %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
taosCloseFile(&fdFrom); tstrerror(code));
taosCloseFile(&fdTo);
} }
taosCloseFile(&fdFrom);
taosCloseFile(&fdTo);
return code; return code;
} }
@ -674,15 +651,21 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
STFileSet *fset = rtner->fset; STFileSet *fset = rtner->fset;
STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
if (!fobj) return code; if (!fobj) {
return 0;
}
int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now); int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
if (expLevel < 0) return code; // expired if (expLevel < 0) { // expired
return 0;
}
SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config; SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
int32_t s3KeepLocal = pCfg->s3KeepLocal; int32_t s3KeepLocal = pCfg->s3KeepLocal;
int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now); int32_t s3ExpLevel = tsdbS3FidLevel(fset->fid, &rtner->tsdb->keepCfg, s3KeepLocal, rtner->now);
if (s3ExpLevel < 1) return code; // keep on local storage if (s3ExpLevel < 1) { // keep on local storage
return 0;
}
int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize; int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->s3ChunkSize;
int32_t lcn = fobj->f->lcn; int32_t lcn = fobj->f->lcn;
@ -698,19 +681,18 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
STimeWindow win = {0}; STimeWindow win = {0};
tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey); tsdbFidKeyRange(fset->fid, rtner->tsdb->keepCfg.days, rtner->tsdb->keepCfg.precision, &win.skey, &win.ekey);
tsdbInfo("vgId:%d, compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); tsdbInfo("vgId:%d, async compact begin lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1); code = tsdbAsyncCompact(rtner->tsdb, &win, pCfg->sttTrigger == 1);
tsdbInfo("vgId:%d, compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn); tsdbInfo("vgId:%d, async compact end lcn: %d.", TD_VID(rtner->tsdb->pVnode), lcn);
goto _exit;
return code; return code;
} }
code = tsdbMigrateDataFileS3(rtner, fobj, size, chunksize); TAOS_CHECK_GOTO(tsdbMigrateDataFileS3(rtner, fobj, size, chunksize), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} else { } else {
if (lcn <= 1) { if (lcn <= 1) {
tsdbError("vgId:%d, incorrect lcn: %d, %s at line %d", TD_VID(rtner->tsdb->pVnode), lcn, __func__, lino); TAOS_CHECK_GOTO(TSDB_CODE_INVALID_PARA, &lino, _exit);
return code;
} }
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1); tsdbTFileLastChunkName(rtner->tsdb, fobj->f, fname1);
@ -720,8 +702,7 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
int64_t size = 0; int64_t size = 0;
taosStatFile(fname1, &size, &mtime, NULL); taosStatFile(fname1, &size, &mtime, NULL);
if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) { if (size > chunksize && mtime < rtner->now - tsS3UploadDelaySec) {
code = tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize); TAOS_CHECK_GOTO(tsdbMigrateDataFileLCS3(rtner, fobj, size, chunksize), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} else { } else {
tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino); tsdbError("vgId:%d, file: %s not found, %s at line %d", TD_VID(rtner->tsdb->pVnode), fname1, __func__, lino);
@ -731,7 +712,8 @@ static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -750,11 +732,15 @@ int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
} }
if (!tsS3Enabled) { if (!tsS3Enabled) {
return code; return 0;
} }
taosThreadMutexLock(&tsdb->mutex); taosThreadMutexLock(&tsdb->mutex);
code = tsdbAsyncRetentionImpl(tsdb, now, true); code = tsdbAsyncRetentionImpl(tsdb, now, true);
taosThreadMutexUnlock(&tsdb->mutex); taosThreadMutexUnlock(&tsdb->mutex);
if (code) {
tsdbError("vgId:%d, %s failed, reason:%s", TD_VID(tsdb->pVnode), __func__, tstrerror(code));
}
return code; return code;
} }

26
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -58,17 +58,9 @@ struct STsdbSnapReader {
}; };
static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) {
int32_t code = 0;
int32_t lino = 0;
TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose);
tsdbDataFileReaderClose(&reader->dataReader); tsdbDataFileReaderClose(&reader->dataReader);
return 0;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;
} }
static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
@ -255,7 +247,6 @@ static int32_t tsdbSnapCmprData(STsdbSnapReader* reader, uint8_t** data) {
SColCompressInfo cmprInfo = {.pColCmpr = NULL, .defaultCmprAlg = NO_COMPRESSION}; SColCompressInfo cmprInfo = {.pColCmpr = NULL, .defaultCmprAlg = NO_COMPRESSION};
code = tBlockDataCompress(reader->blockData, (void*)&cmprInfo, reader->buffers, reader->buffers + 4); code = tBlockDataCompress(reader->blockData, (void*)&cmprInfo, reader->buffers, reader->buffers + 4);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// TSDB_CHECK_CODE(code, lino, _exit);
int32_t size = 0; int32_t size = 0;
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
@ -448,8 +439,8 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type,
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), tsdbError("vgId:%d %s failed at %s:%d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
__func__, lino, tstrerror(code), sver, ever, type); __func__, __FILE__, lino, tstrerror(code), sver, ever, type);
tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr); tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr);
taosMemoryFree(reader[0]); taosMemoryFree(reader[0]);
reader[0] = NULL; reader[0] = NULL;
@ -461,10 +452,11 @@ _exit:
} }
int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) { int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) {
if (reader[0] == NULL) return 0; if (reader[0] == NULL) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb* tsdb = reader[0]->tsdb; STsdb* tsdb = reader[0]->tsdb;
@ -488,12 +480,6 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) {
taosMemoryFree(reader[0]); taosMemoryFree(reader[0]);
reader[0] = NULL; reader[0] = NULL;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
} else {
tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
}
return code; return code;
} }

37
source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
View File

@ -148,30 +148,14 @@ _exit:
} }
static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) { static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) {
int32_t code = 0;
int32_t lino = 0;
TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose); TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose);
return 0;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;
} }
static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) { static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) {
int32_t code = 0;
int32_t lino = 0;
reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr); reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr);
reader->dataIter->idx = 0; reader->dataIter->idx = 0;
return 0;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
}
return code;
} }
static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) { static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) {
@ -392,17 +376,6 @@ _exit:
return code; return code;
} }
static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) {
int32_t code = 0;
int32_t lino = 0;
_exit:
if (code) {
TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
}
return code;
}
static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; } static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; }
static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) { static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) {
@ -445,10 +418,8 @@ static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t
writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr;
int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec()); int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec());
if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) { code = tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did);
code = TSDB_CODE_NO_AVAIL_DISK; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did); tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did);
code = tsdbSnapRAWWriteFileSetOpenWriter(writer); code = tsdbSnapRAWWriteFileSetOpenWriter(writer);

424
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -40,7 +40,9 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
int32_t lino = 0; int32_t lino = 0;
reader[0] = taosMemoryCalloc(1, sizeof(*reader[0])); reader[0] = taosMemoryCalloc(1, sizeof(*reader[0]));
if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (reader[0] == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
reader[0]->config[0] = config[0]; reader[0]->config[0] = config[0];
reader[0]->buffers = config->buffers; reader[0]->buffers = config->buffers;
@ -50,13 +52,11 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
// open file // open file
if (fname) { if (fname) {
code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
char fname1[TSDB_FILENAME_LEN]; char fname1[TSDB_FILENAME_LEN];
tsdbTFileName(config->tsdb, config->file, fname1); tsdbTFileName(config->tsdb, config->file, fname1);
code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// // open each segment reader // // open each segment reader
@ -64,17 +64,17 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
ASSERT(offset >= TSDB_FHDR_SIZE); ASSERT(offset >= TSDB_FHDR_SIZE);
int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
#if 1 #if 1
code = tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, TAOS_CHECK_GOTO(tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
encryptKey); encryptAlgoirthm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
#else #else
int64_t size = config->file->size; int64_t size = config->file->size;
for (; size > TSDB_FHDR_SIZE; size--) { for (; size > TSDB_FHDR_SIZE; size--) {
code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, encryptAlgoirthm, code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
encryptKey); encryptAlgoirthm, encryptKey);
if (code) continue; if (code) continue;
if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size || if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size ||
(*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size || (*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size ||
@ -89,7 +89,8 @@ int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *con
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
tsdbSttFileReaderClose(reader); tsdbSttFileReaderClose(reader);
} }
return code; return code;
@ -118,13 +119,14 @@ int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray *
int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk); int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk);
void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code = tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data,
tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data, reader->footer->statisBlkPtr->size, 0, reader->footer->statisBlkPtr->size, 0, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -149,13 +151,14 @@ int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tom
int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk); int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk);
void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code = tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data,
tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data, reader->footer->tombBlkPtr->size, 0, reader->footer->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -180,13 +183,14 @@ int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBl
int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk); int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk);
void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size); void *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
if (!data) return TSDB_CODE_OUT_OF_MEMORY; if (!data) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
int32_t code = int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size,
tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size, 0, 0, encryptAlgorithm, encryptKey);
encryptAlgorithm, encryptKey);
if (code) { if (code) {
taosMemoryFree(data); taosMemoryFree(data);
return code; return code;
@ -212,20 +216,20 @@ int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk,
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
code = tBlockDataDecompress(&br, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -241,17 +245,16 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
SBuffer *assist = reader->buffers + 2; SBuffer *assist = reader->buffers + 2;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load key part // load key part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode header // decode header
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0); SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
code = tGetDiskDataHdr(&br, &hdr); TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
@ -262,8 +265,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
bData->nRow = hdr.nRow; bData->nRow = hdr.nRow;
// key part // key part
code = tBlockDataDecompressKeyPart(&hdr, &br, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(br.offset == buffer0->size); ASSERT(br.offset == buffer0->size);
bool loadExtra = false; bool loadExtra = false;
@ -280,9 +282,9 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
// load SBlockCol part // load SBlockCol part
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// load each column // load each column
SBlockCol blockCol = { SBlockCol blockCol = {
@ -302,8 +304,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
break; break;
} }
code = tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg); TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (cid < blockCol.cid) { if (cid < blockCol.cid) {
@ -320,27 +321,26 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
.szValue = 0, .szValue = 0,
.offset = 0, .offset = 0,
}; };
code = tBlockDataDecompressColData(&hdr, &none, &br, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &none, &br, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else if (cid == blockCol.cid) { } else if (cid == blockCol.cid) {
// load from file // load from file
tBufferClear(buffer1); tBufferClear(buffer1);
code = TAOS_CHECK_GOTO(
tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset, tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, encryptAlgorithm,
encryptAlgorithm, encryptKey); encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode the buffer // decode the buffer
SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1); SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
code = tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist); TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -353,12 +353,12 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load // load
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, encryptAlgorithm,
encryptAlgorithm, encryptKey); encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode // decode
int32_t size = 0; int32_t size = 0;
@ -372,15 +372,15 @@ int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk
.originalSize = tombBlk->numRec * sizeof(int64_t), .originalSize = tombBlk->numRec * sizeof(int64_t),
.compressedSize = tombBlk->size[i], .compressedSize = tombBlk->size[i],
}; };
code = tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist); TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
br.offset += tombBlk->size[i]; br.offset += tombBlk->size[i];
} }
ASSERT(br.offset == tombBlk->dp->size); ASSERT(br.offset == tombBlk->dp->size);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -393,12 +393,12 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
SBuffer *assist = reader->buffers + 1; SBuffer *assist = reader->buffers + 1;
int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// load data // load data
tBufferClear(buffer0); tBufferClear(buffer0);
code = tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0, TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
// decode data // decode data
tStatisBlockClear(statisBlock); tStatisBlockClear(statisBlock);
@ -413,8 +413,7 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
.originalSize = statisBlk->numRec * sizeof(int64_t), .originalSize = statisBlk->numRec * sizeof(int64_t),
}; };
code = tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist); TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
br.offset += statisBlk->size[i]; br.offset += statisBlk->size[i];
} }
@ -424,25 +423,23 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
// decode compress info // decode compress info
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]); TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]); TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// decode value columns // decode value columns
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist); TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize); br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize);
} }
for (int32_t i = 0; i < statisBlk->numOfPKs; i++) { for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
code = tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist); TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist), &lino,
TSDB_CHECK_CODE(code, lino, _exit); _exit);
br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize); br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize);
} }
} }
@ -451,7 +448,8 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -486,7 +484,7 @@ struct SSttFileWriter {
static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info, static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info,
int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers, int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers,
SVersionRange *range, int32_t encryptAlgorithm, char* encryptKey) { SVersionRange *range, int32_t encryptAlgorithm, char *encryptKey) {
if (blockData->nRow == 0) return 0; if (blockData->nRow == 0) return 0;
int32_t code = 0; int32_t code = 0;
@ -510,25 +508,21 @@ static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, S
} }
tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer); tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer);
code = tBlockDataCompress(blockData, info, buffers, buffers + 4); TAOS_CHECK_RETURN(tBlockDataCompress(blockData, info, buffers, buffers + 4));
if (code) return code;
sttBlk->bInfo.offset = *fileSize; sttBlk->bInfo.offset = *fileSize;
sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size; sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size;
sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey; sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey;
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
if (buffers[i].size) { if (buffers[i].size) {
code = tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey); TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey));
if (code) return code;
*fileSize += buffers[i].size; *fileSize += buffers[i].size;
} }
} }
code = TARRAY2_APPEND_PTR(sttBlkArray, sttBlk); TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(sttBlkArray, sttBlk));
if (code) return code;
tBlockDataClear(blockData); tBlockDataClear(blockData);
return 0; return 0;
} }
@ -543,22 +537,25 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr)); code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr));
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray, TAOS_CHECK_GOTO(
writer->buffers, &writer->ctx->range, tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray,
encryptAlgorithm, encryptKey); writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
taosHashCleanup(info.pColCmpr); taosHashCleanup(info.pColCmpr);
return code; return code;
} }
static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
if (writer->staticBlock->numOfRecords == 0) return 0; if (writer->staticBlock->numOfRecords == 0) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -586,7 +583,7 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
statisBlk.maxTbid.uid = record.uid; statisBlk.maxTbid.uid = record.uid;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
// compress each column // compress each column
for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) { for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) {
@ -597,12 +594,10 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
}; };
tBufferClear(buffer0); tBufferClear(buffer0);
code = tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist); TAOS_CHECK_GOTO(tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(
tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, encryptAlgorithm, encryptKey),
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, &lino, _exit);
encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit);
statisBlk.size[i] = info.compressedSize; statisBlk.size[i] = info.compressedSize;
statisBlk.dp->size += info.compressedSize; statisBlk.dp->size += info.compressedSize;
@ -617,74 +612,72 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
tBufferClear(buffer1); tBufferClear(buffer1);
for (int32_t i = 0; i < statisBlk.numOfPKs; i++) { for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
code = tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist); TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
code = tValueColumnCompressInfoEncode(&compressInfo, buffer0);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int32_t i = 0; i < statisBlk.numOfPKs; i++) { for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
code = tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist); TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
code = tValueColumnCompressInfoEncode(&compressInfo, buffer0);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey),
&lino, _exit);
writer->file->size += buffer0->size; writer->file->size += buffer0->size;
statisBlk.dp->size += buffer0->size; statisBlk.dp->size += buffer0->size;
code = tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey),
&lino, _exit);
writer->file->size += buffer1->size; writer->file->size += buffer1->size;
statisBlk.dp->size += buffer1->size; statisBlk.dp->size += buffer1->size;
} }
code = TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk); TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tStatisBlockClear(writer->staticBlock); tStatisBlockClear(writer->staticBlock);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0; if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) {
return 0;
}
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size, TAOS_CHECK_GOTO(
writer->tombBlkArray, writer->buffers, &writer->ctx->range, tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
encryptAlgorithm, encryptKey); writer->tombBlkArray, writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize, int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
int32_t encryptAlgorithm, char* encryptKey) { int32_t encryptAlgorithm, char *encryptKey) {
ptr->size = TARRAY2_DATA_LEN(sttBlkArray); ptr->size = TARRAY2_DATA_LEN(sttBlkArray);
if (ptr->size > 0) { if (ptr->size > 0) {
ptr->offset = *fileSize; ptr->offset = *fileSize;
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptAlgorithm, TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size,
encryptKey); encryptAlgorithm, encryptKey));
if (code) {
return code;
}
*fileSize += ptr->size; *fileSize += ptr->size;
} }
@ -696,15 +689,16 @@ static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
int32_t lino; int32_t lino;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size, TAOS_CHECK_GOTO(tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
encryptAlgorithm, encryptKey); encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -713,20 +707,21 @@ static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
int32_t code = 0; int32_t code = 0;
int32_t lino; int32_t lino;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray); writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
if (writer->footer->statisBlkPtr->size) { if (writer->footer->statisBlkPtr->size) {
writer->footer->statisBlkPtr->offset = writer->file->size; writer->footer->statisBlkPtr->offset = writer->file->size;
code = tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray), TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey); writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
writer->file->size += writer->footer->statisBlkPtr->size; writer->file->size += writer->footer->statisBlkPtr->size;
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -736,30 +731,31 @@ static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
int32_t lino = 0; int32_t lino = 0;
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr, &writer->file->size, TAOS_CHECK_GOTO(tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr,
encryptAlgorithm, encryptKey); &writer->file->size, encryptAlgorithm, encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm, int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
char* encryptKey) { char *encryptKey) {
int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey); TAOS_CHECK_RETURN(
if (code) return code; tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey));
*fileSize += sizeof(*footer); *fileSize += sizeof(*footer);
return 0; return 0;
} }
static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) { static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) {
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey); return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey);
} }
@ -795,15 +791,13 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(writer->config->tsdb, writer->file, fname); tsdbTFileName(writer->config->tsdb, writer->file, fname);
code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
writer->file->size += sizeof(hdr); writer->file->size += sizeof(hdr);
// range // range
@ -813,7 +807,8 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -843,34 +838,19 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
int32_t lino; int32_t lino;
int32_t code; int32_t code;
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
code = tsdbSttFileDoWriteStatisBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteSttBlk(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlk(writer), &lino, _exit);
TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlk(writer), &lino, _exit);
code = tsdbSttFileDoWriteTombBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteFooter(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbSttFileDoUpdateHeader(writer), &lino, _exit);
code = tsdbSttFileDoWriteSttBlk(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoWriteStatisBlk(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoWriteTombBlk(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoWriteFooter(writer);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSttFileDoUpdateHeader(writer);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit);
tsdbCloseFile(&writer->fd); tsdbCloseFile(&writer->fd);
@ -882,12 +862,12 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
}; };
tsdbTFileUpdVerRange(&op.nf, writer->ctx->range); tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
code = TARRAY2_APPEND(opArray, op); TAOS_CHECK_GOTO(TARRAY2_APPEND(opArray, op), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -902,7 +882,9 @@ static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) {
int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) { int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) {
writer[0] = taosMemoryCalloc(1, sizeof(*writer[0])); writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (writer[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
writer[0]->config[0] = config[0]; writer[0]->config[0] = config[0];
writer[0]->ctx->opened = false; writer[0]->ctx->opened = false;
@ -915,11 +897,9 @@ int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArr
if (writer[0]->ctx->opened) { if (writer[0]->ctx->opened) {
if (abort) { if (abort) {
code = tsdbSttFWriterCloseAbort(writer[0]); TAOS_CHECK_GOTO(tsdbSttFWriterCloseAbort(writer[0]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbSttFWriterCloseCommit(writer[0], opArray); TAOS_CHECK_GOTO(tsdbSttFWriterCloseCommit(writer[0], opArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbSttFWriterDoClose(writer[0]); tsdbSttFWriterDoClose(writer[0]);
} }
@ -928,7 +908,8 @@ int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArr
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -938,20 +919,19 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
int32_t lino = 0; int32_t lino = 0;
if (!writer->ctx->opened) { if (!writer->ctx->opened) {
code = tsdbSttFWriterDoOpen(writer); TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) { if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) {
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb); TAOS_CHECK_GOTO(tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
TABLEID id = {.suid = row->suid, .uid = row->suid ? 0 : row->uid}; TABLEID id = {
code = tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0); .suid = row->suid,
TSDB_CHECK_CODE(code, lino, _exit); .uid = row->suid ? 0 : row->uid,
};
TAOS_CHECK_GOTO(tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0), &lino, _exit);
} }
if (writer->ctx->tbid->uid != row->uid) { if (writer->ctx->tbid->uid != row->uid) {
@ -965,8 +945,7 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
for (;;) { for (;;) {
code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow); code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow);
if (code == TSDB_CODE_INVALID_PARA) { if (code == TSDB_CODE_INVALID_PARA) {
code = tsdbSttFileDoWriteStatisBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
continue; continue;
} else { } else {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -975,9 +954,9 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
} }
if (row->row.type == TSDBROW_ROW_FMT) { if (row->row.type == TSDBROW_ROW_FMT) {
code = tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, // TAOS_CHECK_GOTO(tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid, //
TSDBROW_SVERSION(&row->row), writer->config->skmRow); TSDBROW_SVERSION(&row->row), writer->config->skmRow),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
} }
// row to col conversion // row to col conversion
@ -989,21 +968,20 @@ int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
&& tsdbRowCompareWithoutVersion(&row->row, && tsdbRowCompareWithoutVersion(&row->row,
&tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0 // &tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0 //
) { ) {
code = tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema); TAOS_CHECK_GOTO(tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
if (writer->blockData->nRow >= writer->config->maxRow) { if (writer->blockData->nRow >= writer->config->maxRow) {
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid); TAOS_CHECK_GOTO(tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -1018,13 +996,13 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) {
row->uid = bdata->uid ? bdata->uid : bdata->aUid[i]; row->uid = bdata->uid ? bdata->uid : bdata->aUid[i];
row->row = tsdbRowFromBlockData(bdata, i); row->row = tsdbRowFromBlockData(bdata, i);
code = tsdbSttFileWriteRow(writer, row); TAOS_CHECK_GOTO(tsdbSttFileWriteRow(writer, row), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} }
return code; return code;
} }
@ -1034,31 +1012,27 @@ int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *re
int32_t lino; int32_t lino;
if (!writer->ctx->opened) { if (!writer->ctx->opened) {
code = tsdbSttFWriterDoOpen(writer); TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
if (writer->blockData->nRow > 0) { if (writer->blockData->nRow > 0) {
code = tsdbSttFileDoWriteBlockData(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) { if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) {
code = tsdbSttFileDoWriteStatisBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = tTombBlockPut(writer->tombBlock, record); TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) { if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) {
code = tsdbSttFileDoWriteTombBlock(writer); TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
tstrerror(code));
} else { } else {
tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64 tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
", version:%" PRId64, ", version:%" PRId64,
@ -1068,4 +1042,4 @@ _exit:
return code; return code;
} }
bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; } bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; }

201
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -58,12 +58,10 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
// read SBlockIdx array // read SBlockIdx array
if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { if ((ctx->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbReadBlockIdx(reader, ctx->aBlockIdx); TAOS_CHECK_GOTO(tsdbReadBlockIdx(reader, ctx->aBlockIdx), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(ctx->aBlockIdx) > 0) { if (taosArrayGetSize(ctx->aBlockIdx) > 0) {
// init/open file fd // init/open file fd
@ -77,22 +75,19 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// open fd // open fd
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
tsdbTFileName(tsdb, &file, fname); tsdbTFileName(tsdb, &file, fname);
code = tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// convert // convert
for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) { for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(ctx->aBlockIdx); ++iBlockIdx) {
SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx); SBlockIdx *pBlockIdx = taosArrayGet(ctx->aBlockIdx, iBlockIdx);
code = tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk); TAOS_CHECK_GOTO(tsdbReadDataBlk(reader, pBlockIdx, ctx->mDataBlk), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) { for (int32_t iDataBlk = 0; iDataBlk < ctx->mDataBlk->nItem; ++iDataBlk) {
SDataBlk dataBlk[1]; SDataBlk dataBlk[1];
@ -134,8 +129,7 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
record.count = 0; record.count = 0;
} }
code = tBrinBlockPut(ctx->brinBlock, &record); TAOS_CHECK_GOTO(tBrinBlockPut(ctx->brinBlock, &record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (ctx->brinBlock->numOfRecords >= ctx->maxRow) { if (ctx->brinBlock->numOfRecords >= ctx->maxRow) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
@ -149,28 +143,28 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords > 0) { if (ctx->brinBlock->numOfRecords > 0) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, TAOS_CHECK_GOTO(
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
TSDB_CHECK_CODE(code, lino, _exit); ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey),
&lino, _exit);
} }
code = tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr, TAOS_CHECK_GOTO(tsdbFileWriteBrinBlk(ctx->fd, ctx->brinBlkArray, ctx->footer->brinBlkPtr,
&fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey); &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey),
TSDB_CHECK_CODE(code, lino, _exit); &lino, _exit);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm, code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
TARRAY2_DESTROY(ctx->brinBlkArray, NULL); TARRAY2_DESTROY(ctx->brinBlkArray, NULL);
tBrinBlockDestroy(ctx->brinBlock); tBrinBlockDestroy(ctx->brinBlock);
@ -201,12 +195,11 @@ static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -229,12 +222,11 @@ static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -246,12 +238,10 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); SArray *aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (aSttBlk == NULL) { if (aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbReadSttBlk(reader, iStt, aSttBlk); TAOS_CHECK_GOTO(tsdbReadSttBlk(reader, iStt, aSttBlk), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(aSttBlk) > 0) { if (taosArrayGetSize(aSttBlk) > 0) {
SSttFile *pSttF = pDFileSet->aSttF[iStt]; SSttFile *pSttF = pDFileSet->aSttF[iStt];
@ -279,31 +269,27 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
.minVer = VERSION_MAX, .minVer = VERSION_MAX,
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, &fobj); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, &fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0); TAOS_CHECK_GOTO(tsdbOpenFile(fobj->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE, &ctx->fd, 0), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) { for (int32_t iSttBlk = 0; iSttBlk < taosArrayGetSize(aSttBlk); iSttBlk++) {
code = TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)); TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(ctx->sttBlkArray, (SSttBlk *)taosArrayGet(aSttBlk, iSttBlk)), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
} }
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm, code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit1); tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey), &lino,
_exit);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
code = TARRAY2_APPEND(lvl->fobjArr, fobj); TAOS_CHECK_GOTO(TARRAY2_APPEND(lvl->fobjArr, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit1);
_exit1: _exit1:
TARRAY2_DESTROY(ctx->sttBlkArray, NULL); TARRAY2_DESTROY(ctx->sttBlkArray, NULL);
@ -311,7 +297,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
taosArrayDestroy(aSttBlk); taosArrayDestroy(aSttBlk);
return code; return code;
@ -326,24 +312,21 @@ static int32_t tsdbUpgradeStt(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r
} }
SSttLvl *lvl; SSttLvl *lvl;
code = tsdbSttLvlInit(0, &lvl); TAOS_CHECK_GOTO(tsdbSttLvlInit(0, &lvl), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) { for (int32_t iStt = 0; iStt < pDFileSet->nSttF; ++iStt) {
code = tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl); TAOS_CHECK_GOTO(tsdbUpgradeSttFile(tsdb, pDFileSet, reader, fset, iStt, lvl), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (TARRAY2_SIZE(lvl->fobjArr) > 0) { if (TARRAY2_SIZE(lvl->fobjArr) > 0) {
code = TARRAY2_APPEND(fset->lvlArr, lvl); TAOS_CHECK_GOTO(TARRAY2_APPEND(fset->lvlArr, lvl), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
tsdbSttLvlClear(&lvl); tsdbSttLvlClear(&lvl);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -357,40 +340,33 @@ static int32_t tsdbUpgradeFileSet(STsdb *tsdb, SDFileSet *pDFileSet, TFileSetArr
SDataFReader *reader; SDataFReader *reader;
STFileSet *fset; STFileSet *fset;
code = tsdbTFileSetInit(pDFileSet->fid, &fset); TAOS_CHECK_GOTO(tsdbTFileSetInit(pDFileSet->fid, &fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDataFReaderOpen(&reader, tsdb, pDFileSet); TAOS_CHECK_GOTO(tsdbDataFReaderOpen(&reader, tsdb, pDFileSet), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .head // .head
code = tsdbUpgradeHead(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeHead(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .data // .data
code = tsdbUpgradeData(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeData(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .sma // .sma
code = tsdbUpgradeSma(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeSma(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// .stt // .stt
if (pDFileSet->nSttF > 0) { if (pDFileSet->nSttF > 0) {
code = tsdbUpgradeStt(tsdb, pDFileSet, reader, fset); TAOS_CHECK_GOTO(tsdbUpgradeStt(tsdb, pDFileSet, reader, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
tsdbDataFReaderClose(&reader); tsdbDataFReaderClose(&reader);
code = TARRAY2_APPEND(fileSetArray, fset); TAOS_CHECK_GOTO(TARRAY2_APPEND(fileSetArray, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbInfo("vgId:%d upgrade file set end, fid:%d", TD_VID(tsdb->pVnode), pDFileSet->fid); tsdbInfo("vgId:%d upgrade file set end, fid:%d", TD_VID(tsdb->pVnode), pDFileSet->fid);
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -412,8 +388,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, fobj); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
fset->farr[TSDB_FTYPE_TOMB] = *fobj; fset->farr[TSDB_FTYPE_TOMB] = *fobj;
} else { // to .stt file } else { // to .stt file
@ -430,28 +405,26 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
.maxVer = VERSION_MIN, .maxVer = VERSION_MIN,
}; };
code = tsdbTFileObjInit(tsdb, &file, fobj); TAOS_CHECK_GOTO(tsdbTFileObjInit(tsdb, &file, fobj), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = TARRAY2_APPEND(lvl->fobjArr, fobj[0]); TAOS_CHECK_GOTO(TARRAY2_APPEND(lvl->fobjArr, fobj[0]), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
char fname[TSDB_FILENAME_LEN] = {0}; char fname[TSDB_FILENAME_LEN] = {0};
code = tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0); TAOS_CHECK_GOTO(
TSDB_CHECK_CODE(code, lino, _exit); tsdbOpenFile(fobj[0]->fname, tsdb, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CREATE, fd, 0), &lino,
_exit);
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char *encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); TAOS_CHECK_GOTO(tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
fobj[0]->f->size += TSDB_FHDR_SIZE; fobj[0]->f->size += TSDB_FHDR_SIZE;
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -489,15 +462,13 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey); tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &ctx->minKey, &ctx->maxKey);
if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { if ((ctx->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx); SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(aDelIdx, iDelIdx);
code = tsdbReadDelData(reader, pDelIdx, ctx->aDelData); TAOS_CHECK_GOTO(tsdbReadDelData(reader, pDelIdx, ctx->aDelData), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) { for (int32_t iDelData = 0; iDelData < taosArrayGetSize(ctx->aDelData); iDelData++) {
SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData); SDelData *pDelData = (SDelData *)taosArrayGet(ctx->aDelData, iDelData);
@ -510,31 +481,30 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
.ekey = pDelData->eKey, .ekey = pDelData->eKey,
}; };
code = tTombBlockPut(ctx->tombBlock, &record); TAOS_CHECK_GOTO(tTombBlockPut(ctx->tombBlock, &record), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) { if (TOMB_BLOCK_SIZE(ctx->tombBlock) > ctx->maxRow) {
if (ctx->fd == NULL) { if (ctx->fd == NULL) {
code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TAOS_CHECK_GOTO(tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, TAOS_CHECK_GOTO(
ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
TSDB_CHECK_CODE(code, lino, _exit); ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey),
&lino, _exit);
} }
} }
} }
if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) { if (TOMB_BLOCK_SIZE(ctx->tombBlock) > 0) {
if (ctx->fd == NULL) { if (ctx->fd == NULL) {
code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt); TAOS_CHECK_GOTO(tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray, TAOS_CHECK_GOTO(
ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
TSDB_CHECK_CODE(code, lino, _exit); ctx->buffers, &tombRange, ctx->encryptAlgorithm, ctx->encryptKey),
&lino, _exit);
} }
if (ctx->fd != NULL) { if (ctx->fd != NULL) {
@ -556,15 +526,14 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); TAOS_CHECK_GOTO(tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
tsdbCloseFile(&ctx->fd); tsdbCloseFile(&ctx->fd);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
for (int32_t i = 0; i < ARRAY_SIZE(ctx->buffers); i++) { for (int32_t i = 0; i < ARRAY_SIZE(ctx->buffers); i++) {
tBufferDestroy(ctx->buffers + i); tBufferDestroy(ctx->buffers + i);
@ -583,27 +552,23 @@ static int32_t tsdbUpgradeTombFile(STsdb *tsdb, SDelFile *pDelFile, TFileSetArra
SArray *aDelIdx = NULL; SArray *aDelIdx = NULL;
if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { if ((aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbDelFReaderOpen(&reader, pDelFile, tsdb); TAOS_CHECK_GOTO(tsdbDelFReaderOpen(&reader, pDelFile, tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadDelIdx(reader, aDelIdx); TAOS_CHECK_GOTO(tsdbReadDelIdx(reader, aDelIdx), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(aDelIdx) > 0) { if (taosArrayGetSize(aDelIdx) > 0) {
STFileSet *fset; STFileSet *fset;
TARRAY2_FOREACH(fileSetArray, fset) { TARRAY2_FOREACH(fileSetArray, fset) {
code = tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset); TAOS_CHECK_GOTO(tsdbDumpTombDataToFSet(tsdb, reader, aDelIdx, fset), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
tsdbDelFReaderClose(&reader); tsdbDelFReaderClose(&reader);
taosArrayDestroy(aDelIdx); taosArrayDestroy(aDelIdx);
@ -616,19 +581,17 @@ static int32_t tsdbDoUpgradeFileSystem(STsdb *tsdb, TFileSetArray *fileSetArray)
// upgrade each file set // upgrade each file set
for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) { for (int32_t i = 0; i < taosArrayGetSize(tsdb->fs.aDFileSet); i++) {
code = tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray); TAOS_CHECK_GOTO(tsdbUpgradeFileSet(tsdb, taosArrayGet(tsdb->fs.aDFileSet, i), fileSetArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
// upgrade tomb file // upgrade tomb file
if (tsdb->fs.pDelFile != NULL) { if (tsdb->fs.pDelFile != NULL) {
code = tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray); TAOS_CHECK_GOTO(tsdbUpgradeTombFile(tsdb, tsdb->fs.pDelFile, fileSetArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
if (code) { if (code) {
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
} }
return code; return code;
} }
@ -640,24 +603,21 @@ static int32_t tsdbUpgradeFileSystem(STsdb *tsdb, int8_t rollback) {
TFileSetArray fileSetArray[1] = {0}; TFileSetArray fileSetArray[1] = {0};
// open old file system // open old file system
code = tsdbFSOpen(tsdb, rollback); TAOS_CHECK_GOTO(tsdbFSOpen(tsdb, rollback), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDoUpgradeFileSystem(tsdb, fileSetArray); TAOS_CHECK_GOTO(tsdbDoUpgradeFileSystem(tsdb, fileSetArray), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// close file system // close file system
code = tsdbFSClose(tsdb); TAOS_CHECK_GOTO(tsdbFSClose(tsdb), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
// save new file system // save new file system
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
current_fname(tsdb, fname, TSDB_FCURRENT); current_fname(tsdb, fname, TSDB_FCURRENT);
code = save_fs(fileSetArray, fname); TAOS_CHECK_GOTO(save_fs(fileSetArray, fname), &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) { if (code) {
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
} }
TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear); TARRAY2_DESTROY(fileSetArray, tsdbTFileSetClear);
@ -670,9 +630,8 @@ int32_t tsdbCheckAndUpgradeFileSystem(STsdb *tsdb, int8_t rollback) {
tsdbGetCurrentFName(tsdb, fname, NULL); tsdbGetCurrentFName(tsdb, fname, NULL);
if (!taosCheckExistFile(fname)) return 0; if (!taosCheckExistFile(fname)) return 0;
int32_t code = tsdbUpgradeFileSystem(tsdb, rollback); TAOS_CHECK_RETURN(tsdbUpgradeFileSystem(tsdb, rollback));
if (code) return code;
taosRemoveFile(fname); (void)taosRemoveFile(fname);
return 0; return 0;
} }

34
source/dnode/vnode/src/tsdb/tsdbWrite.c
View File

@ -28,6 +28,7 @@ int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400
// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) {
int32_t code;
int32_t arrSize = 0; int32_t arrSize = 0;
int32_t affectedrows = 0; int32_t affectedrows = 0;
int32_t numOfRows = 0; int32_t numOfRows = 0;
@ -39,18 +40,16 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2
arrSize = taosArrayGetSize(pMsg->aSubmitTbData); arrSize = taosArrayGetSize(pMsg->aSubmitTbData);
// scan and convert // scan and convert
if ((terrno = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) { if ((code = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) {
if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno)); tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno));
} }
return -1; return code;
} }
// loop to insert // loop to insert
for (int32_t i = 0; i < arrSize; ++i) { for (int32_t i = 0; i < arrSize; ++i) {
if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { TAOS_CHECK_RETURN(tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows));
return -1;
}
} }
if (pRsp != NULL) { if (pRsp != NULL) {
@ -74,22 +73,12 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowK
} }
int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t code = 0;
STsdbKeepCfg *pCfg = &pTsdb->keepCfg; STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
TSKEY now = taosGetTimestamp(pCfg->precision); TSKEY now = taosGetTimestamp(pCfg->precision);
TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];
int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);
/*
int32_t nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
if (nlevel > 1 && tsS3Enabled) {
if (nlevel == 3) {
minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1;
} else if (nlevel == 2) {
minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep0;
}
}
*/
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i);
if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
@ -99,22 +88,17 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
int32_t nRows = aColData[0].nVal; int32_t nRows = aColData[0].nVal;
TSKEY *aKey = (TSKEY *)aColData[0].pData; TSKEY *aKey = (TSKEY *)aColData[0].pData;
for (int32_t r = 0; r < nRows; ++r) { for (int32_t r = 0; r < nRows; ++r) {
if ((code = tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)) < 0) { TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now));
goto _exit;
}
} }
} }
} else { } else {
int32_t nRows = taosArrayGetSize(pData->aRowP); int32_t nRows = taosArrayGetSize(pData->aRowP);
for (int32_t r = 0; r < nRows; ++r) { for (int32_t r = 0; r < nRows; ++r) {
SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r);
if ((code = tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)) < 0) { TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now));
goto _exit;
}
} }
} }
} }
_exit: return 0;
return code;
} }

5
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -458,11 +458,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
if (tqCommit(pVnode->pTq) < 0) {
code = TSDB_CODE_FAILED;
TSDB_CHECK_CODE(code, lino, _exit);
}
// commit info // commit info
if (vnodeCommitInfo(dir) < 0) { if (vnodeCommitInfo(dir) < 0) {
code = terrno; code = terrno;

58
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -59,9 +59,9 @@ struct SVSnapReader {
int8_t tqHandleDone; int8_t tqHandleDone;
STqSnapReader *pTqSnapReader; STqSnapReader *pTqSnapReader;
int8_t tqOffsetDone; int8_t tqOffsetDone;
STqOffsetReader *pTqOffsetReader; STqSnapReader *pTqOffsetReader;
int8_t tqCheckInfoDone; int8_t tqCheckInfoDone;
STqCheckInfoReader *pTqCheckInfoReader; STqSnapReader *pTqCheckInfoReader;
// stream // stream
int8_t streamTaskDone; int8_t streamTaskDone;
SStreamTaskReader *pStreamTaskReader; SStreamTaskReader *pStreamTaskReader;
@ -233,11 +233,11 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
} }
if (pReader->pTqOffsetReader) { if (pReader->pTqOffsetReader) {
tqOffsetReaderClose(&pReader->pTqOffsetReader); tqSnapReaderClose(&pReader->pTqOffsetReader);
} }
if (pReader->pTqCheckInfoReader) { if (pReader->pTqCheckInfoReader) {
tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader); tqSnapReaderClose(&pReader->pTqCheckInfoReader);
} }
taosMemoryFree(pReader); taosMemoryFree(pReader);
@ -365,7 +365,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
vInfo("vgId:%d tq transform start", vgId); vInfo("vgId:%d tq transform start", vgId);
if (!pReader->tqHandleDone) { if (!pReader->tqHandleDone) {
if (pReader->pTqSnapReader == NULL) { if (pReader->pTqSnapReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqSnapReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE, &pReader->pTqSnapReader);
if (code < 0) goto _err; if (code < 0) goto _err;
} }
@ -384,11 +384,11 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
} }
if (!pReader->tqCheckInfoDone) { if (!pReader->tqCheckInfoDone) {
if (pReader->pTqCheckInfoReader == NULL) { if (pReader->pTqCheckInfoReader == NULL) {
code = tqCheckInfoReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqCheckInfoReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO, &pReader->pTqCheckInfoReader);
if (code < 0) goto _err; if (code < 0) goto _err;
} }
code = tqCheckInfoRead(pReader->pTqCheckInfoReader, ppData); code = tqSnapRead(pReader->pTqCheckInfoReader, ppData);
if (code) { if (code) {
goto _err; goto _err;
} else { } else {
@ -396,18 +396,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
goto _exit; goto _exit;
} else { } else {
pReader->tqCheckInfoDone = 1; pReader->tqCheckInfoDone = 1;
code = tqCheckInfoReaderClose(&pReader->pTqCheckInfoReader); code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
if (code) goto _err; if (code) goto _err;
} }
} }
} }
if (!pReader->tqOffsetDone) { if (!pReader->tqOffsetDone) {
if (pReader->pTqOffsetReader == NULL) { if (pReader->pTqOffsetReader == NULL) {
code = tqOffsetReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, &pReader->pTqOffsetReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET, &pReader->pTqOffsetReader);
if (code < 0) goto _err; if (code < 0) goto _err;
} }
code = tqOffsetSnapRead(pReader->pTqOffsetReader, ppData); code = tqSnapRead(pReader->pTqOffsetReader, ppData);
if (code) { if (code) {
goto _err; goto _err;
} else { } else {
@ -415,7 +415,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
goto _exit; goto _exit;
} else { } else {
pReader->tqOffsetDone = 1; pReader->tqOffsetDone = 1;
code = tqOffsetReaderClose(&pReader->pTqOffsetReader); code = tqSnapReaderClose(&pReader->pTqOffsetReader);
if (code) goto _err; if (code) goto _err;
} }
} }
@ -536,9 +536,9 @@ struct SVSnapWriter {
// tsdb raw // tsdb raw
STsdbSnapRAWWriter *pTsdbSnapRAWWriter; STsdbSnapRAWWriter *pTsdbSnapRAWWriter;
// tq // tq
STqSnapWriter *pTqSnapWriter; STqSnapWriter *pTqSnapHandleWriter;
STqOffsetWriter *pTqOffsetWriter; STqSnapWriter *pTqSnapOffsetWriter;
STqCheckInfoWriter *pTqCheckInfoWriter; STqSnapWriter *pTqSnapCheckInfoWriter;
// stream // stream
SStreamTaskWriter *pStreamTaskWriter; SStreamTaskWriter *pStreamTaskWriter;
SStreamStateWriter *pStreamStateWriter; SStreamStateWriter *pStreamStateWriter;
@ -736,18 +736,18 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
if (code) goto _exit; if (code) goto _exit;
} }
if (pWriter->pTqSnapWriter) { if (pWriter->pTqSnapHandleWriter) {
code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback); code = tqSnapWriterClose(&pWriter->pTqSnapHandleWriter, rollback);
if (code) goto _exit; if (code) goto _exit;
} }
if (pWriter->pTqCheckInfoWriter) { if (pWriter->pTqSnapCheckInfoWriter) {
code = tqCheckInfoWriterClose(&pWriter->pTqCheckInfoWriter, rollback); code = tqSnapWriterClose(&pWriter->pTqSnapCheckInfoWriter, rollback);
if (code) goto _exit; if (code) goto _exit;
} }
if (pWriter->pTqOffsetWriter) { if (pWriter->pTqSnapOffsetWriter) {
code = tqOffsetWriterClose(&pWriter->pTqOffsetWriter, rollback); code = tqSnapWriterClose(&pWriter->pTqSnapOffsetWriter, rollback);
if (code) goto _exit; if (code) goto _exit;
} }
@ -872,32 +872,32 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
} break; } break;
case SNAP_DATA_TQ_HANDLE: { case SNAP_DATA_TQ_HANDLE: {
// tq handle // tq handle
if (pWriter->pTqSnapWriter == NULL) { if (pWriter->pTqSnapHandleWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter);
if (code) goto _err; if (code) goto _err;
} }
code = tqSnapWrite(pWriter->pTqSnapWriter, pData, nData); code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData);
if (code) goto _err; if (code) goto _err;
} break; } break;
case SNAP_DATA_TQ_CHECKINFO: { case SNAP_DATA_TQ_CHECKINFO: {
// tq checkinfo // tq checkinfo
if (pWriter->pTqCheckInfoWriter == NULL) { if (pWriter->pTqSnapCheckInfoWriter == NULL) {
code = tqCheckInfoWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqCheckInfoWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter);
if (code) goto _err; if (code) goto _err;
} }
code = tqCheckInfoWrite(pWriter->pTqCheckInfoWriter, pData, nData); code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData);
if (code) goto _err; if (code) goto _err;
} break; } break;
case SNAP_DATA_TQ_OFFSET: { case SNAP_DATA_TQ_OFFSET: {
// tq offset // tq offset
if (pWriter->pTqOffsetWriter == NULL) { if (pWriter->pTqSnapOffsetWriter == NULL) {
code = tqOffsetWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqOffsetWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter);
if (code) goto _err; if (code) goto _err;
} }
code = tqOffsetSnapWrite(pWriter->pTqOffsetWriter, pData, nData); code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData);
if (code) goto _err; if (code) goto _err;
} break; } break;
case SNAP_DATA_STREAM_TASK: case SNAP_DATA_STREAM_TASK:

16
source/libs/catalog/inc/catalogInt.h
View File

@ -25,6 +25,7 @@ extern "C" {
#include "tcommon.h" #include "tcommon.h"
#include "ttimer.h" #include "ttimer.h"
#include "tglobal.h" #include "tglobal.h"
#include "os.h"
#define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6
#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
@ -669,12 +670,12 @@ typedef struct SCtgCacheItemInfo {
#define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE)
#define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE)
#define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) #define CTG_QUEUE_INC() (void)atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
#define CTG_QUEUE_DEC() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) #define CTG_QUEUE_DEC() (void)atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
#define CTG_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) #define CTG_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n)
#define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) #define CTG_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n)
#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) #define CTG_STAT_GET(_item) (void)atomic_load_64(&(_item))
#define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) #define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n))
#define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) #define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n))
@ -971,7 +972,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq
int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx,
int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); int32_t* fetchIdx, int32_t baseResIdx, SArray* pList);
void* ctgCloneDbCfgInfo(void* pSrc); int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst);
int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action);
int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation);
@ -1112,7 +1113,7 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName);
int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user); int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user);
int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup,
bool* exists); bool* exists);
SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName);
int32_t ctgdGetOneHandle(SCatalog** pHandle); int32_t ctgdGetOneHandle(SCatalog** pHandle);
int ctgVgInfoComp(const void* lp, const void* rp); int ctgVgInfoComp(const void* lp, const void* rp);
int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo);
@ -1165,6 +1166,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn,
void* bInput); void* bInput);
int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag, int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag,
CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName); CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName);
void ctgFreeTask(SCtgTask* pTask, bool freeRes);
extern SCatalogMgmt gCtgMgmt; extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug; extern SCtgDebug gCTGDebug;

212
source/libs/catalog/src/catalog.c
View File

@ -86,7 +86,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char*
if (code) { if (code) {
if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) {
ctgDebug("db no longer exist, dbFName:%s, dbId:0x%" PRIx64, input.db, input.dbId); ctgDebug("db no longer exist, dbFName:%s, dbId:0x%" PRIx64, input.db, input.dbId);
ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId); CTG_ERR_RET(ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId));
} }
CTG_ERR_RET(code); CTG_ERR_RET(code);
@ -116,8 +116,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx*
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName));
CTG_ERR_JRET( CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL));
ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL));
} else if (CTG_FLAG_IS_STB(ctx->flag)) { } else if (CTG_FLAG_IS_STB(ctx->flag)) {
ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName));
@ -128,8 +127,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx*
CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL));
} }
} else { } else {
ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag);
ctx->flag);
// if get from vnode failed or no table meta, will not try mnode // if get from vnode failed or no table meta, will not try mnode
CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL));
@ -166,7 +164,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx*
if (CTG_IS_META_NULL(output->metaType)) { if (CTG_IS_META_NULL(output->metaType)) {
ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName));
ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false));
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
} }
@ -216,7 +214,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
} }
if (CTG_IS_META_BOTH(output->metaType)) { if (CTG_IS_META_BOTH(output->metaType)) {
memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); TAOS_MEMCPY(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta));
*pTableMeta = output->tbMeta; *pTableMeta = output->tbMeta;
goto _return; goto _return;
@ -233,7 +231,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
taosMemoryFreeClear(output->tbMeta); taosMemoryFreeClear(output->tbMeta);
SName stbName = *ctx->pName; SName stbName = *ctx->pName;
strcpy(stbName.tname, output->tbName); TAOS_STRCPY(stbName.tname, output->tbName);
SCtgTbMetaCtx stbCtx = {0}; SCtgTbMetaCtx stbCtx = {0};
stbCtx.flag = ctx->flag; stbCtx.flag = ctx->flag;
stbCtx.pName = &stbName; stbCtx.pName = &stbName;
@ -244,7 +242,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
continue; continue;
} }
memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); TAOS_MEMCPY(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta));
break; break;
} }
@ -254,15 +252,15 @@ _return:
if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) { if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) {
char dbFName[TSDB_DB_FNAME_LEN] = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0};
if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { if (CTG_FLAG_IS_SYS_DB(ctx->flag)) {
strcpy(dbFName, ctx->pName->dbname); TAOS_STRCPY(dbFName, ctx->pName->dbname);
} else { } else {
tNameGetFullDbName(ctx->pName, dbFName); (void)tNameGetFullDbName(ctx->pName, dbFName);
} }
if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) { if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) {
ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); (void)ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); // already in error
} else { } else {
ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); (void)ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); // already in error
} }
} }
@ -285,18 +283,18 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) {
int32_t code = 0; int32_t code = 0;
strcpy(output->dbFName, rspMsg->dbFName); TAOS_STRCPY(output->dbFName, rspMsg->dbFName);
output->dbId = rspMsg->dbId; output->dbId = rspMsg->dbId;
if (TSDB_CHILD_TABLE == rspMsg->tableType && NULL == rspMsg->pSchemas) { if (TSDB_CHILD_TABLE == rspMsg->tableType && NULL == rspMsg->pSchemas) {
strcpy(output->ctbName, rspMsg->tbName); TAOS_STRCPY(output->ctbName, rspMsg->tbName);
SET_META_TYPE_CTABLE(output->metaType); SET_META_TYPE_CTABLE(output->metaType);
CTG_ERR_JRET(queryCreateCTableMetaFromMsg(rspMsg, &output->ctbMeta)); CTG_ERR_JRET(queryCreateCTableMetaFromMsg(rspMsg, &output->ctbMeta));
} else { } else {
strcpy(output->tbName, rspMsg->tbName); TAOS_STRCPY(output->tbName, rspMsg->tbName);
SET_META_TYPE_TABLE(output->metaType); SET_META_TYPE_TABLE(output->metaType);
@ -348,14 +346,14 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq,
_return: _return:
ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); (void)ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); // cache update not fatal error
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) {
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName); (void)tNameGetFullDbName(pTableName, dbFName);
CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType)); CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType));
if (*tbType > 0) { if (*tbType > 0) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -454,7 +452,11 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.type = TSDB_DATA_TYPE_JSON;
tagVal.pData = pJson; tagVal.pData = pJson;
tagVal.nData = strlen(pJson); tagVal.nData = strlen(pJson);
taosArrayPush(pTagVals, &tagVal); if (NULL == taosArrayPush(pTagVals, &tagVal)) {
taosMemoryFree(pJson);
taosArrayDestroy(pTagVals);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
} else { } else {
CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals)); CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals));
} }
@ -484,7 +486,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTabl
CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &tbMeta)); CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &tbMeta));
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db); (void)tNameGetFullDbName(pTableName, db);
SHashObj* vgHash = NULL; SHashObj* vgHash = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, NULL)); CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, NULL));
@ -555,7 +557,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName*
SCtgDBCache* dbCache = NULL; SCtgDBCache* dbCache = NULL;
int32_t code = 0; int32_t code = 0;
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db); (void)tNameGetFullDbName(pTableName, db);
SDBVgInfo* vgInfo = NULL; SDBVgInfo* vgInfo = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, exists)); CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, exists));
@ -590,7 +592,7 @@ int32_t ctgGetTbsHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctI
SCtgDBCache* dbCache = NULL; SCtgDBCache* dbCache = NULL;
int32_t code = 0; int32_t code = 0;
char dbFName[TSDB_DB_FNAME_LEN] = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0};
snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); (void)snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb);
SDBVgInfo* vgInfo = NULL; SDBVgInfo* vgInfo = NULL;
CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL)); CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL));
@ -615,7 +617,7 @@ _return:
int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) {
int32_t code = 0; int32_t code = 0;
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db); (void)tNameGetFullDbName(pTableName, db);
SCtgDBCache *dbCache = NULL; SCtgDBCache *dbCache = NULL;
SCtgTbCache *tbCache = NULL; SCtgTbCache *tbCache = NULL;
@ -688,8 +690,8 @@ _return:
void ctgProcessTimerEvent(void *param, void *tmrId) { void ctgProcessTimerEvent(void *param, void *tmrId) {
CTG_API_NENTER(); CTG_API_NENTER();
ctgdShowCacheInfo(); (void)ctgdShowCacheInfo();
ctgdShowStatInfo(); (void)ctgdShowStatInfo();
int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize);
if (cacheMaxSize >= 0) { if (cacheMaxSize >= 0) {
@ -703,7 +705,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) {
int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false);
if (code) { if (code) {
qError("clear cache enqueue failed, error:%s", tstrerror(code)); qError("clear cache enqueue failed, error:%s", tstrerror(code));
taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer);
} }
goto _return; goto _return;
@ -711,7 +713,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) {
} }
qTrace("reset catalog timer"); qTrace("reset catalog timer");
taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer);
_return: _return:
@ -723,10 +725,8 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName
if (pDbCfg->cfgVersion < 0) { if (pDbCfg->cfgVersion < 0) {
CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL));
SDbCfgInfo *pCfg = ctgCloneDbCfgInfo(pDbCfg); SDbCfgInfo *pCfg = NULL;
if (NULL == pCfg) { CTG_ERR_RET(ctgCloneDbCfgInfo(pDbCfg, &pCfg));
return TSDB_CODE_OUT_OF_MEMORY;
}
CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false));
} }
@ -735,6 +735,59 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName
} }
int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) {
STableTSMAInfoRsp tsmasRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
*ppRes = tsmasRsp.pTsmas;
tsmasRsp.pTsmas = NULL;
for (int32_t i = 0; i < (*ppRes)->size; ++i) {
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false));
}
return TSDB_CODE_SUCCESS;
_return:
if (tsmasRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmasRsp);
}
CTG_RET(code);
}
int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
STableTSMAInfoRsp tsmaRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1);
*pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0);
taosArrayDestroy(tsmaRsp.pTsmas);
tsmaRsp.pTsmas = NULL;
_return:
if (tsmaRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmaRsp);
}
CTG_RET(code);
}
int32_t catalogInit(SCatalogCfg* cfg) { int32_t catalogInit(SCatalogCfg* cfg) {
qDebug("catalogInit start"); qDebug("catalogInit start");
if (gCtgMgmt.pCluster) { if (gCtgMgmt.pCluster) {
@ -742,10 +795,10 @@ int32_t catalogInit(SCatalogCfg* cfg) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); TAOS_MEMSET(&gCtgMgmt, 0, sizeof(gCtgMgmt));
if (cfg) { if (cfg) {
memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); TAOS_MEMCPY(&gCtgMgmt.cfg, cfg, sizeof(*cfg));
if (gCtgMgmt.cfg.maxDBCacheNum == 0) { if (gCtgMgmt.cfg.maxDBCacheNum == 0) {
gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
@ -1129,7 +1182,7 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp) {
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy(pIndex, pRsp, sizeof(STableIndex)); TAOS_MEMCPY(pIndex, pRsp, sizeof(STableIndex));
CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false));
@ -1247,17 +1300,26 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray*
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t code = TSDB_CODE_SUCCESS;
SName name = {0}; SName name = {0};
int32_t sver = 0; int32_t sver = 0;
int32_t tver = 0; int32_t tver = 0;
int32_t tbNum = taosArrayGetSize(pTables); int32_t tbNum = taosArrayGetSize(pTables);
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i);
if (NULL == pTb) {
ctgError("fail to get the %dth table, tbNum:%d", i, tbNum);
CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) {
continue; continue;
} }
tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); if (tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) {
ctgError("invalid tbFName format, tbFName:%s, idx:%d", pTb->tbFName, i);
CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (IS_SYS_DBNAME(name.dbname)) { if (IS_SYS_DBNAME(name.dbname)) {
continue; continue;
@ -1266,18 +1328,18 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray*
int32_t tbType = 0; int32_t tbType = 0;
uint64_t suid = 0; uint64_t suid = 0;
char stbName[TSDB_TABLE_FNAME_LEN]; char stbName[TSDB_TABLE_FNAME_LEN];
ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName); CTG_ERR_JRET(ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName));
if ((sver >= 0 && sver < pTb->sver) || (tver >= 0 && tver < pTb->tver)) { if ((sver >= 0 && sver < pTb->sver) || (tver >= 0 && tver < pTb->tver)) {
switch (tbType) { switch (tbType) {
case TSDB_CHILD_TABLE: { case TSDB_CHILD_TABLE: {
SName stb = name; SName stb = name;
tstrncpy(stb.tname, stbName, sizeof(stb.tname)); tstrncpy(stb.tname, stbName, sizeof(stb.tname));
ctgRemoveTbMeta(pCtg, &stb); CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &stb));
break; break;
} }
case TSDB_SUPER_TABLE: case TSDB_SUPER_TABLE:
case TSDB_NORMAL_TABLE: case TSDB_NORMAL_TABLE:
ctgRemoveTbMeta(pCtg, &name); CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &name));
break; break;
default: default:
ctgError("ignore table type %d", tbType); ctgError("ignore table type %d", tbType);
@ -1286,7 +1348,9 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray*
} }
} }
CTG_API_LEAVE(TSDB_CODE_SUCCESS); _return:
CTG_API_LEAVE(code);
} }
int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) { int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) {
@ -1454,10 +1518,10 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SC
_return: _return:
if (pJob) { if (pJob) {
taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); (void)taosReleaseRef(gCtgMgmt.jobPool, pJob->refId);
if (code) { if (code) {
taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId);
} }
} }
@ -1558,7 +1622,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
while (pAuth != NULL) { while (pAuth != NULL) {
size_t len = 0; size_t len = 0;
void* key = taosHashGetKey(pAuth, &len); void* key = taosHashGetKey(pAuth, &len);
strncpy((*users)[i].user, key, len); TAOS_STRNCPY((*users)[i].user, key, len);
(*users)[i].user[len] = 0; (*users)[i].user[len] = 0;
(*users)[i].version = pAuth->userAuth.version; (*users)[i].version = pAuth->userAuth.version;
++i; ++i;
@ -1795,10 +1859,12 @@ int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma, int32_t
if (!pCtg || !ppTsma) { if (!pCtg || !ppTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t code = 0; int32_t code = 0;
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false)); CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false));
_return: _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
@ -1807,10 +1873,12 @@ int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma) {
if (!pCtg || !pTsma) { if (!pCtg || !pTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
} }
int32_t code = 0; int32_t code = 0;
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true)); CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true));
_return: _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
@ -1823,36 +1891,14 @@ int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma) {
} }
if (!pCtg->dbCache) { if (!pCtg->dbCache) {
return TSDB_CODE_SUCCESS;
}
CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true));
_return:
CTG_API_LEAVE(code);
}
int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) {
STableTSMAInfoRsp tsmasRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return; goto _return;
} }
CTG_ERR_JRET(code);
assert(tsmasRsp.pTsmas); CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true));
assert(tsmasRsp.pTsmas->size > 0);
*ppRes = tsmasRsp.pTsmas;
tsmasRsp.pTsmas = NULL;
for (int32_t i = 0; i < (*ppRes)->size; ++i) {
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false));
}
return TSDB_CODE_SUCCESS;
_return: _return:
if (tsmasRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmasRsp); CTG_API_LEAVE(code);
}
CTG_RET(code);
} }
int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
@ -1870,27 +1916,6 @@ _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
STableTSMAInfoRsp tsmaRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1);
*pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0);
taosArrayDestroy(tsmaRsp.pTsmas);
tsmaRsp.pTsmas = NULL;
_return:
if (tsmaRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmaRsp);
}
CTG_RET(code);
}
int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
CTG_API_ENTER(); CTG_API_ENTER();
@ -1903,6 +1928,7 @@ int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTs
CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma)); CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma));
_return: _return:
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
} }
@ -1930,7 +1956,7 @@ void catalogDestroy(void) {
} }
if (gCtgMgmt.cacheTimer) { if (gCtgMgmt.cacheTimer) {
taosTmrStop(gCtgMgmt.cacheTimer); (void)taosTmrStop(gCtgMgmt.cacheTimer);
gCtgMgmt.cacheTimer = NULL; gCtgMgmt.cacheTimer = NULL;
taosTmrCleanUp(gCtgMgmt.timer); taosTmrCleanUp(gCtgMgmt.timer);
gCtgMgmt.timer = NULL; gCtgMgmt.timer = NULL;
@ -1939,8 +1965,8 @@ void catalogDestroy(void) {
atomic_store_8((int8_t*)&gCtgMgmt.exit, true); atomic_store_8((int8_t*)&gCtgMgmt.exit, true);
if (!taosCheckCurrentInDll()) { if (!taosCheckCurrentInDll()) {
ctgClearCacheEnqueue(NULL, false, true, true, true); (void)ctgClearCacheEnqueue(NULL, false, true, true, true);
taosThreadJoin(gCtgMgmt.updateThread, NULL); (void)taosThreadJoin(gCtgMgmt.updateThread, NULL);
} }
taosHashCleanup(gCtgMgmt.pCluster); taosHashCleanup(gCtgMgmt.pCluster);

1656
source/libs/catalog/src/ctgAsync.c
View File

File diff suppressed because it is too large Load Diff

1239
source/libs/catalog/src/ctgCache.c
View File

File diff suppressed because it is too large Load Diff

241
source/libs/catalog/src/ctgRemote.c
View File

@ -57,8 +57,23 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
for (int32_t i = 0; i < taskNum; ++i) { for (int32_t i = 0; i < taskNum; ++i) {
int32_t* taskId = taosArrayGet(cbParam->taskId, i); int32_t* taskId = taosArrayGet(cbParam->taskId, i);
if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i);
if (NULL == msgIdx) {
ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId);
if (NULL == pTask) {
ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (msgNum > 0) { if (msgNum > 0) {
pRsp = taosArrayGet(batchRsp.pRsps, i); pRsp = taosArrayGet(batchRsp.pRsps, i);
@ -89,12 +104,17 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu
tReq.pTask = pTask; tReq.pTask = pTask;
tReq.msgIdx = pRsp->msgIdx; tReq.msgIdx = pRsp->msgIdx;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq.msgIdx); SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq.msgIdx);
if (NULL == pMsgCtx) {
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq.msgIdx, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pMsgCtx->pBatchs = pBatchs; pMsgCtx->pBatchs = pBatchs;
ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId,
pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs);
(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); (void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal
} }
CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs));
@ -398,7 +418,16 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode));
} else { } else {
int32_t* taskId = taosArrayGet(cbParam->taskId, 0); int32_t* taskId = taosArrayGet(cbParam->taskId, 0);
if (NULL == taskId) {
ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId);
if (NULL == pTask) {
ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId,
TMSG_INFO(cbParam->reqType + 1)); TMSG_INFO(cbParam->reqType + 1));
@ -412,6 +441,11 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) {
} }
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1);
if (NULL == pMsgCtx) {
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pMsgCtx->pBatchs = pBatchs; pMsgCtx->pBatchs = pBatchs;
#endif #endif
@ -432,7 +466,7 @@ _return:
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
if (pJob) { if (pJob) {
taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); (void)taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId);
} }
CTG_API_LEAVE(code); CTG_API_LEAVE(code);
@ -450,6 +484,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArr
SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam));
if (NULL == param) { if (NULL == param) {
qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam));
taosMemoryFree(msgSendInfo);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
} }
@ -482,7 +517,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob,
SMsgSendInfo* pMsgSendInfo = NULL; SMsgSendInfo* pMsgSendInfo = NULL;
CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo)); CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo));
ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); CTG_ERR_JRET(ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId));
pMsgSendInfo->requestId = pConn->requestId; pMsgSendInfo->requestId = pConn->requestId;
pMsgSendInfo->requestObjRefId = pConn->requestObjRefId; pMsgSendInfo->requestObjRefId = pConn->requestObjRefId;
@ -515,18 +550,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
void* msg, uint32_t msgSize) { void* msg, uint32_t msgSize) {
int32_t code = 0; int32_t code = 0;
SCtgTask* pTask = tReq->pTask; SCtgTask* pTask = tReq->pTask;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
SCtgBatch newBatch = {0}; SCtgBatch newBatch = {0};
SBatchMsg req = {0}; SBatchMsg req = {0};
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
if (NULL == pMsgCtx) {
ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SHashObj* pBatchs = pMsgCtx->pBatchs;
SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId));
if (NULL == pBatch) { if (NULL == pBatch) {
newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg));
newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t));
newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t));
if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) { if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) {
taosArrayDestroy(newBatch.pMsgs);
taosArrayDestroy(newBatch.pTaskIds);
taosArrayDestroy(newBatch.pMsgIdxs);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
} }
@ -556,12 +598,16 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) {
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
pName = ctgGetFetchName(ctx->pNames, fetch); CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){ } else if (CTG_TASK_GET_TB_TSMA == pTask->type){
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else { } else {
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
pName = ctx->pName; pName = ctx->pName;
@ -569,14 +615,26 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else { } else {
ctgError("invalid vnode msgType %d", msgType); ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR); CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
} }
tNameGetFullDbName(pName, newBatch.dbFName); (void)tNameGetFullDbName(pName, newBatch.dbFName);
} }
newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META;
@ -616,12 +674,16 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) {
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
pName = ctgGetFetchName(ctx->pNames, fetch); CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName));
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){ } else if (CTG_TASK_GET_TB_TSMA == pTask->type){
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else { } else {
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
pName = ctx->pName; pName = ctx->pName;
@ -629,14 +691,26 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
if (NULL == pTbReq) {
ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (NULL == pName) {
ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
} else { } else {
ctgError("invalid vnode msgType %d", msgType); ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR); CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
} }
tNameGetFullDbName(pName, pBatch->dbFName); (void)tNameGetFullDbName(pName, pBatch->dbFName);
} }
ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId,
@ -752,7 +826,10 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -765,7 +842,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL));
@ -801,7 +878,10 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -814,7 +894,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL));
@ -854,7 +934,10 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -867,7 +950,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db));
@ -909,7 +992,10 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -922,7 +1008,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName));
@ -964,7 +1050,10 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -977,7 +1066,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName));
@ -993,7 +1082,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n
int32_t reqType = TDMT_MND_GET_TABLE_INDEX; int32_t reqType = TDMT_MND_GET_TABLE_INDEX;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(name, tbFName); (void)tNameExtractFullName(name, tbFName);
ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName);
@ -1021,7 +1110,10 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1034,7 +1126,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
@ -1076,7 +1168,10 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1089,7 +1184,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName));
@ -1131,7 +1226,10 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1144,7 +1242,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user));
@ -1162,7 +1260,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t reqType = TDMT_MND_TABLE_META; int32_t reqType = TDMT_MND_TABLE_META;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
sprintf(tbFName, "%s.%s", dbFName, tbName); (void)sprintf(tbFName, "%s.%s", dbFName, tbName);
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName);
@ -1188,7 +1286,10 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1201,7 +1302,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName));
@ -1213,7 +1314,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const
int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out, int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out,
SCtgTaskReq* tReq) { SCtgTaskReq* tReq) {
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName); (void)tNameGetFullDbName(pTableName, dbFName);
return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq); return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq);
} }
@ -1222,10 +1323,10 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa
STableMetaOutput* out, SCtgTaskReq* tReq) { STableMetaOutput* out, SCtgTaskReq* tReq) {
SCtgTask* pTask = tReq ? tReq->pTask : NULL; SCtgTask* pTask = tReq ? tReq->pTask : NULL;
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName); (void)tNameGetFullDbName(pTableName, dbFName);
int32_t reqType = TDMT_VND_TABLE_META; int32_t reqType = TDMT_VND_TABLE_META;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); (void)sprintf(tbFName, "%s.%s", dbFName, pTableName->tname);
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse];
@ -1261,12 +1362,15 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa
#else #else
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(ctx->pName, dbFName); (void)tNameGetFullDbName(ctx->pName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen));
#endif #endif
@ -1279,7 +1383,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName));
@ -1294,10 +1398,10 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t reqType = TDMT_VND_TABLE_CFG; int32_t reqType = TDMT_VND_TABLE_CFG;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFName); (void)tNameExtractFullName(pTableName, tbFName);
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName); (void)tNameGetFullDbName(pTableName, dbFName);
SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname};
SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse];
@ -1325,12 +1429,15 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
#else #else
SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx;
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(ctx->pName, dbFName); (void)tNameGetFullDbName(ctx->pName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg,
msgLen)); msgLen));
@ -1344,7 +1451,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
@ -1359,10 +1466,10 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t reqType = TDMT_MND_TABLE_CFG; int32_t reqType = TDMT_MND_TABLE_CFG;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFName); (void)tNameExtractFullName(pTableName, tbFName);
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName); (void)tNameGetFullDbName(pTableName, dbFName);
SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname};
ctgDebug("try to get table cfg from mnode, tbFName:%s", tbFName); ctgDebug("try to get table cfg from mnode, tbFName:%s", tbFName);
@ -1386,7 +1493,10 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1399,7 +1509,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
@ -1435,7 +1545,10 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1448,7 +1561,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL));
@ -1465,7 +1578,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
SCtgTask* pTask = tReq ? tReq->pTask : NULL; SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char fullName[TSDB_TABLE_FNAME_LEN]; char fullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pName, fullName); (void)tNameExtractFullName(pName, fullName);
ctgDebug("try to get view info from mnode, viewFName:%s", fullName); ctgDebug("try to get view info from mnode, viewFName:%s", fullName);
@ -1490,7 +1603,10 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1503,7 +1619,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, fullName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, fullName));
@ -1519,7 +1635,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa
SCtgTask* pTask = tReq ? tReq->pTask : NULL; SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(name, tbFName); (void)tNameExtractFullName(name, tbFName);
ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName);
@ -1544,7 +1660,10 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif #endif
@ -1557,7 +1676,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
@ -1573,7 +1692,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS; int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTbName, tbFName); (void)tNameExtractFullName(pTbName, tbFName);
SCtgTask* pTask = tReq ? tReq->pTask : NULL; SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
@ -1602,15 +1721,17 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen));
#else #else
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTbName, dbFName); (void)tNameGetFullDbName(pTbName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) { if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
taosArrayPush(pTaskId, &pTask->taskId); if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) {
taosArrayDestroy(pTaskId);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_RET( CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
#endif #endif
} }
@ -1621,7 +1742,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c
}; };
SRpcMsg rpcRsp = {0}; SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp));
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));

17
source/libs/catalog/src/ctgRent.c
View File

@ -100,7 +100,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
memcpy(orig, meta, size); TAOS_MEMCPY(orig, meta, size);
qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
@ -185,8 +185,12 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_
} }
void *meta = taosArrayGet(slot->meta, 0); void *meta = taosArrayGet(slot->meta, 0);
if (NULL == meta) {
qError("get the 0th meta in slot failed, total:%d", (int32_t)metaNum);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
memcpy(*res, meta, msize); TAOS_MEMCPY(*res, meta, msize);
*num = (uint32_t)metaNum; *num = (uint32_t)metaNum;
@ -227,13 +231,14 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) {
return; return;
} }
int32_t code = TSDB_CODE_SUCCESS;
void *pIter = taosHashIterate(dbCache->stbCache, NULL); void *pIter = taosHashIterate(dbCache->stbCache, NULL);
while (pIter) { while (pIter) {
uint64_t *suid = NULL; uint64_t *suid = NULL;
suid = taosHashGetKey(pIter, NULL); suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS == code = ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare);
ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { if (TSDB_CODE_SUCCESS == code) {
ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid); ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid);
} }
@ -265,6 +270,7 @@ void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) {
void* pIter = taosHashIterate(dbCache->tsmaCache, NULL); void* pIter = taosHashIterate(dbCache->tsmaCache, NULL);
while (pIter) { while (pIter) {
SCtgTSMACache* pCtgCache = pIter; SCtgTSMACache* pCtgCache = pIter;
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
int32_t size = (pCtgCache && pCtgCache->pTsmas) ? pCtgCache->pTsmas->size : 0; int32_t size = (pCtgCache && pCtgCache->pTsmas) ? pCtgCache->pTsmas->size : 0;
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
@ -274,6 +280,7 @@ void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) {
} }
} }
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
pIter = taosHashIterate(dbCache->tsmaCache, pIter); pIter = taosHashIterate(dbCache->tsmaCache, pIter);
} }
} }
@ -325,8 +332,10 @@ int32_t ctgUpdateRentTSMAVersion(SCatalog *pCtg, char *dbFName, const STSMACache
tstrncpy(tsmaRent.name, pTsmaInfo->name, TSDB_TABLE_NAME_LEN); tstrncpy(tsmaRent.name, pTsmaInfo->name, TSDB_TABLE_NAME_LEN);
tstrncpy(tsmaRent.dbFName, dbFName, TSDB_DB_FNAME_LEN); tstrncpy(tsmaRent.dbFName, dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(tsmaRent.tbName, pTsmaInfo->tb, TSDB_TABLE_NAME_LEN); tstrncpy(tsmaRent.tbName, pTsmaInfo->tb, TSDB_TABLE_NAME_LEN);
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->tsmaRent, &tsmaRent, tsmaRent.tsmaId, sizeof(STSMAVersion), CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->tsmaRent, &tsmaRent, tsmaRent.tsmaId, sizeof(STSMAVersion),
ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare)); ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare));
ctgDebug("db %s, 0x%" PRIx64 " tsma %s, 0x%" PRIx64 "version %d updated to tsmaRent", dbFName, tsmaRent.dbId, ctgDebug("db %s, 0x%" PRIx64 " tsma %s, 0x%" PRIx64 "version %d updated to tsmaRent", dbFName, tsmaRent.dbId,
pTsmaInfo->name, pTsmaInfo->tsmaId, pTsmaInfo->version); pTsmaInfo->name, pTsmaInfo->tsmaId, pTsmaInfo->version);

326
source/libs/catalog/src/ctgUtil.c
View File

@ -107,6 +107,10 @@ char* ctgTaskTypeStr(CTG_TASK_TYPE type) {
return "[get table tag]"; return "[get table tag]";
case CTG_TASK_GET_VIEW: case CTG_TASK_GET_VIEW:
return "[get view]"; return "[get view]";
case CTG_TASK_GET_TB_TSMA:
return "[get table TSMA]";
case CTG_TASK_GET_TSMA:
return "[get TSMA]";
default: default:
return "unknown"; return "unknown";
} }
@ -443,10 +447,10 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN
continue; continue;
} }
taosHashRemove(dbCache->tbCache, key, len); (void)taosHashRemove(dbCache->tbCache, key, len);
cacheSize =
len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex);
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
*pClearedSize += cacheSize; *pClearedSize += cacheSize;
(*pCleardNum)++; (*pCleardNum)++;
@ -508,10 +512,10 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache); ctgFreeInstUserCache(pCtg->userCache);
ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); (void)ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo));
ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); (void)ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion));
ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)); (void)ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion));
ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)); (void)ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion));
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false,
HASH_ENTRY_LOCK); HASH_ENTRY_LOCK);
@ -525,7 +529,7 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum);
} }
memset(pCtg->cacheStat.cacheNum, 0, sizeof(pCtg->cacheStat.cacheNum)); TAOS_MEMSET(pCtg->cacheStat.cacheNum, 0, sizeof(pCtg->cacheStat.cacheNum));
CTG_STAT_RT_INC(numOfOpClearCache, 1); CTG_STAT_RT_INC(numOfOpClearCache, 1);
@ -668,7 +672,7 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) {
void ctgResetTbMetaTask(SCtgTask* pTask) { void ctgResetTbMetaTask(SCtgTask* pTask) {
SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx;
memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); TAOS_MEMSET(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo));
taskCtx->flag = CTG_FLAG_UNKNOWN_STB; taskCtx->flag = CTG_FLAG_UNKNOWN_STB;
if (pTask->msgCtx.lastOut) { if (pTask->msgCtx.lastOut) {
@ -1087,7 +1091,10 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) {
} }
} }
taosArrayPush(pCtxs, &ctx); if (NULL == taosArrayPush(pCtxs, &ctx)) {
ctgFreeMsgCtx(&ctx);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1174,13 +1181,14 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(pTableName, db); (void)tNameGetFullDbName(pTableName, db);
if (IS_SYS_DBNAME(pTableName->dbname)) { if (IS_SYS_DBNAME(pTableName->dbname)) {
pVgroup->vgId = MNODE_HANDLE; pVgroup->vgId = MNODE_HANDLE;
if (pMgmtEps) { if (pMgmtEps) {
memcpy(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); TAOS_MEMCPY(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet));
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1191,7 +1199,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d
SVgroupInfo* vgInfo = NULL; SVgroupInfo* vgInfo = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN]; char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName); (void)tNameExtractFullName(pTableName, tbFullName);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix); dbInfo->hashPrefix, dbInfo->hashSuffix);
@ -1243,28 +1251,41 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR
SVgroupInfo mgmtInfo = {0}; SVgroupInfo mgmtInfo = {0};
mgmtInfo.vgId = MNODE_HANDLE; mgmtInfo.vgId = MNODE_HANDLE;
if (pMgmgEpSet) { if (pMgmgEpSet) {
memcpy(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); TAOS_MEMCPY(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet));
} }
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo));
if (NULL == vgInfo) { if (NULL == vgInfo) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy(vgInfo, &mgmtInfo, sizeof(mgmtInfo)); TAOS_MEMCPY(vgInfo, &mgmtInfo, sizeof(mgmtInfo));
ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps,
vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
if (update) { if (update) {
SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
if (NULL == pFetch) {
ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pRes->pRes = vgInfo; pRes->pRes = vgInfo;
} else { } else {
res.pRes = vgInfo; res.pRes = vgInfo;
taosArrayPush(pCtx->pResList, &res); if (NULL == taosArrayPush(pCtx->pResList, &res)) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1281,18 +1302,35 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
*vgInfo = *(SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); SVgroupInfo* pSrcVg = (SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0);
if (NULL == pSrcVg) {
ctgError("fail to get the 0th SVgroupInfo, total:%d", (int32_t)taosArrayGetSize(dbInfo->vgArray));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
TAOS_MEMCPY(vgInfo, pSrcVg, sizeof(*pSrcVg));
ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps,
vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
if (update) { if (update) {
SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
if (NULL == pRes) {
ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pRes->pRes = vgInfo; pRes->pRes = vgInfo;
} else { } else {
res.pRes = vgInfo; res.pRes = vgInfo;
taosArrayPush(pCtx->pResList, &res); if (NULL == taosArrayPush(pCtx->pResList, &res)) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
} }
@ -1300,16 +1338,20 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR
} }
char tbFullName[TSDB_TABLE_FNAME_LEN]; char tbFullName[TSDB_TABLE_FNAME_LEN];
sprintf(tbFullName, "%s.", dbFName); (void)sprintf(tbFullName, "%s.", dbFName);
int32_t offset = strlen(tbFullName); int32_t offset = strlen(tbFullName);
SName* pName = NULL; SName* pName = NULL;
int32_t tbNameLen = 0; int32_t tbNameLen = 0;
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
pName = taosArrayGet(pNames, i); pName = taosArrayGet(pNames, i);
if (NULL == pName) {
ctgError("fail to get the %dth SName, total:%d", i, (int32_t)taosArrayGetSize(pNames));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
tbNameLen = offset + strlen(pName->tname); tbNameLen = offset + strlen(pName->tname);
strcpy(tbFullName + offset, pName->tname); TAOS_STRCPY(tbFullName + offset, pName->tname);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix); dbInfo->hashPrefix, dbInfo->hashSuffix);
@ -1334,11 +1376,22 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR
if (update) { if (update) {
SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx);
if (NULL == pFetch) {
ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i);
if (NULL == pRes) {
ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pRes->pRes = pNewVg; pRes->pRes = pNewVg;
} else { } else {
res.pRes = pNewVg; res.pRes = pNewVg;
taosArrayPush(pCtx->pResList, &res); if (NULL == taosArrayPush(pCtx->pResList, &res)) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
} }
@ -1351,7 +1404,6 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam
CTG_ERR_RET(ctgMakeVgArray(dbInfo)); CTG_ERR_RET(ctgMakeVgArray(dbInfo));
int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); int32_t vgNum = taosArrayGetSize(dbInfo->vgArray);
if (vgNum <= 0) { if (vgNum <= 0) {
ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
@ -1359,11 +1411,11 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam
SVgroupInfo* vgInfo = NULL; SVgroupInfo* vgInfo = NULL;
char tbFullName[TSDB_TABLE_FNAME_LEN]; char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); (void)snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName);
int32_t offset = strlen(tbFullName); int32_t offset = strlen(tbFullName);
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); (void)snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]);
uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod,
dbInfo->hashPrefix, dbInfo->hashSuffix); dbInfo->hashPrefix, dbInfo->hashSuffix);
@ -1475,7 +1527,11 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
void* pIter = taosHashIterate(dbInfo->vgHash, NULL); void* pIter = taosHashIterate(dbInfo->vgHash, NULL);
while (pIter) { while (pIter) {
taosArrayPush(dbInfo->vgArray, pIter); if (NULL == taosArrayPush(dbInfo->vgArray, pIter)) {
taosHashCancelIterate(dbInfo->vgHash, pIter);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pIter = taosHashIterate(dbInfo->vgHash, pIter); pIter = taosHashIterate(dbInfo->vgHash, pIter);
} }
@ -1494,7 +1550,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy(*dst, src, sizeof(SDBVgInfo)); TAOS_MEMCPY(*dst, src, sizeof(SDBVgInfo));
size_t hashSize = taosHashGetSize(src->vgHash); size_t hashSize = taosHashGetSize(src->vgHash);
(*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
@ -1522,6 +1578,11 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) {
if (src->vgArray) { if (src->vgArray) {
(*dst)->vgArray = taosArrayDup(src->vgArray, NULL); (*dst)->vgArray = taosArrayDup(src->vgArray, NULL);
if (NULL == (*dst)->vgArray) {
taosHashCleanup((*dst)->vgHash);
taosMemoryFreeClear(*dst);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1534,7 +1595,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput)
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy(*pOutput, output, sizeof(STableMetaOutput)); TAOS_MEMCPY(*pOutput, output, sizeof(STableMetaOutput));
if (output->tbMeta) { if (output->tbMeta) {
int32_t metaSize = CTG_META_SIZE(output->tbMeta); int32_t metaSize = CTG_META_SIZE(output->tbMeta);
@ -1542,6 +1603,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput)
if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) {
schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt);
} }
(*pOutput)->tbMeta = taosMemoryMalloc(metaSize + schemaExtSize); (*pOutput)->tbMeta = taosMemoryMalloc(metaSize + schemaExtSize);
qDebug("tbMeta cloned, size:%d, p:%p", metaSize, (*pOutput)->tbMeta); qDebug("tbMeta cloned, size:%d, p:%p", metaSize, (*pOutput)->tbMeta);
if (NULL == (*pOutput)->tbMeta) { if (NULL == (*pOutput)->tbMeta) {
@ -1550,10 +1612,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput)
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize);
if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) {
(*pOutput)->tbMeta->schemaExt = (SSchemaExt *)((char *)(*pOutput)->tbMeta + metaSize); (*pOutput)->tbMeta->schemaExt = (SSchemaExt *)((char *)(*pOutput)->tbMeta + metaSize);
memcpy((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize);
} else { } else {
(*pOutput)->tbMeta->schemaExt = NULL; (*pOutput)->tbMeta->schemaExt = NULL;
} }
@ -1576,8 +1638,18 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) {
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
STableIndexInfo* pInfo = taosArrayGet(pIndex, i); STableIndexInfo* pInfo = taosArrayGet(pIndex, i);
if (NULL == pInfo) {
qError("fail to get the %dth STableIndexInfo, total:%d", i, (int32_t)taosArrayGetSize(pIndex));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pInfo = taosArrayPush(*pRes, pInfo); pInfo = taosArrayPush(*pRes, pInfo);
if (NULL == pInfo) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pInfo->expr = taosStrdup(pInfo->expr); pInfo->expr = taosStrdup(pInfo->expr);
if (NULL == pInfo->expr) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1588,6 +1660,9 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, cha
pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.type = TARGET_TYPE_VNODE;
pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.vgId = vgId;
pMsgSendInfo->target.dbFName = taosStrdup(dbFName); pMsgSendInfo->target.dbFName = taosStrdup(dbFName);
if (NULL == pMsgSendInfo->target.dbFName) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} else { } else {
pMsgSendInfo->target.type = TARGET_TYPE_MNODE; pMsgSendInfo->target.type = TARGET_TYPE_MNODE;
} }
@ -1604,6 +1679,11 @@ int32_t ctgGetTablesReqNum(SArray* pList) {
int32_t n = taosArrayGetSize(pList); int32_t n = taosArrayGetSize(pList);
for (int32_t i = 0; i < n; ++i) { for (int32_t i = 0; i < n; ++i) {
STablesReq* pReq = taosArrayGet(pList, i); STablesReq* pReq = taosArrayGet(pList, i);
if (NULL == pReq) {
qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pList));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
total += taosArrayGetSize(pReq->pTables); total += taosArrayGetSize(pReq->pTables);
} }
@ -1613,6 +1693,9 @@ int32_t ctgGetTablesReqNum(SArray* pList) {
int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag) { int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag) {
if (NULL == (*pFetchs)) { if (NULL == (*pFetchs)) {
*pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch)); *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch));
if (NULL == *pFetchs) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
SCtgFetch fetch = {0}; SCtgFetch fetch = {0};
@ -1622,28 +1705,63 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fet
fetch.resIdx = resIdx; fetch.resIdx = resIdx;
fetch.flag = flag; fetch.flag = flag;
taosArrayPush(*pFetchs, &fetch); if (NULL == taosArrayPush(*pFetchs, &fetch)) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) {
STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx);
return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); if (NULL == pReq) {
qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables));
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
*ppName = (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx);
if (NULL == *ppName) {
qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables));
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
} }
static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static int32_t ctgCloneDbVgroup(void* pSrc, void** ppDst) {
#if 0
if (NULL == pSrc) {
*ppDst = NULL;
return TSDB_CODE_SUCCESS;
}
*ppDst = taosArrayDup((const SArray*)pSrc, NULL);
return (*ppDst) ? TSDB_CODE_SUCCESS : TSDB_CODE_OUT_OF_MEMORY;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
}
static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
void* ctgCloneDbCfgInfo(void* pSrc) { int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) {
SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo));
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return terrno;
} }
memcpy(pDst, pSrc, sizeof(SDbCfgInfo));
pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo));
return pDst; if (((SDbCfgInfo *)pSrc)->pRetensions) {
pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL);
if (NULL == pDst->pRetensions) {
taosMemoryFree(pDst);
return terrno;
}
}
*ppDst = pDst;
return TSDB_CODE_SUCCESS;
} }
static void ctgFreeDbCfgInfo(void* p) { static void ctgFreeDbCfgInfo(void* p) {
@ -1651,13 +1769,19 @@ static void ctgFreeDbCfgInfo(void* p) {
freeDbCfgInfo(pDst); freeDbCfgInfo(pDst);
} }
static void* ctgCloneDbInfo(void* pSrc) { static int32_t ctgCloneDbInfo(void* pSrc, void** ppDst) {
#if 0
SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo));
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(pDst, pSrc, sizeof(SDbInfo));
return pDst; TAOS_MEMCPY(pDst, pSrc, sizeof(SDbInfo));
return TSDB_CODE_SUCCESS;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
} }
static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
@ -1690,55 +1814,83 @@ static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void ctgFreeTableMeta(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeTableMeta(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneVgroupInfo(void* pSrc) { static int32_t ctgCloneVgroupInfo(void* pSrc, void** ppDst) {
#if 0
SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo)); SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo));
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;
} }
memcpy(pDst, pSrc, sizeof(SVgroupInfo)); memcpy(pDst, pSrc, sizeof(SVgroupInfo));
return pDst; return pDst;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
} }
static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneTableIndexs(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static int32_t ctgCloneTableIndexs(void* pSrc, void** ppDst) {
#if 0
return taosArrayDup((const SArray*)pSrc, NULL);
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
}
static void ctgFreeTableIndexs(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } static void ctgFreeTableIndexs(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneFuncInfo(void* pSrc) { static int32_t ctgCloneFuncInfo(void* pSrc, void** ppDst) {
#if 0
SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo)); SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo));
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;
} }
memcpy(pDst, pSrc, sizeof(SFuncInfo)); memcpy(pDst, pSrc, sizeof(SFuncInfo));
return pDst; return pDst;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
} }
static void ctgFreeFuncInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeFuncInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneIndexInfo(void* pSrc) { static int32_t ctgCloneIndexInfo(void* pSrc) {
#if 0
SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo)); SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo));
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;
} }
memcpy(pDst, pSrc, sizeof(SIndexInfo)); memcpy(pDst, pSrc, sizeof(SIndexInfo));
return pDst; return pDst;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
} }
static void ctgFreeIndexInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeIndexInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneUserAuth(void* pSrc) { static int32_t ctgCloneUserAuth(void* pSrc) {
#if 0
bool* pDst = taosMemoryMalloc(sizeof(bool)); bool* pDst = taosMemoryMalloc(sizeof(bool));
if (NULL == pDst) { if (NULL == pDst) {
return NULL; return NULL;
} }
*pDst = *(bool*)pSrc; *pDst = *(bool*)pSrc;
return pDst; return pDst;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
} }
static void ctgFreeUserAuth(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeUserAuth(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneQnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static int32_t ctgCloneQnodeList(void* pSrc) {
#if 0
return taosArrayDup((const SArray*)pSrc, NULL);
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
}
static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
@ -1753,11 +1905,18 @@ static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)
static void ctgFreeTableCfg(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeTableCfg(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static int32_t ctgCloneDnodeList(void* pSrc) {
#if 0
return taosArrayDup((const SArray*)pSrc, NULL);
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
}
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneViewMeta(void* pSrc) { static int32_t ctgCloneViewMeta(void* pSrc) {
#if 0
SViewMeta* pSrcMeta = pSrc; SViewMeta* pSrcMeta = pSrc;
SViewMeta* pDst = taosMemoryMalloc(sizeof(SViewMeta)); SViewMeta* pDst = taosMemoryMalloc(sizeof(SViewMeta));
if (NULL == pDst) { if (NULL == pDst) {
@ -1771,6 +1930,9 @@ static void* ctgCloneViewMeta(void* pSrc) {
} }
memcpy(pDst->pSchema, pSrcMeta->pSchema, pSrcMeta->numOfCols * sizeof(*pSrcMeta->pSchema)); memcpy(pDst->pSchema, pSrcMeta->pSchema, pSrcMeta->numOfCols * sizeof(*pSrcMeta->pSchema));
return pDst; return pDst;
#else
return TSDB_CODE_CTG_INTERNAL_ERROR;
#endif
} }
static void ctgFreeViewMeta(void* p) { static void ctgFreeViewMeta(void* p) {
@ -1798,8 +1960,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameExtractFullName(&req->pRawReq->tbName, tbFName); (void)tNameExtractFullName(&req->pRawReq->tbName, tbFName);
tNameGetFullDbName(&req->pRawReq->tbName, dbFName); (void)tNameGetFullDbName(&req->pRawReq->tbName, dbFName);
while (true) { while (true) {
taosMemoryFreeClear(pMeta); taosMemoryFreeClear(pMeta);
@ -1851,7 +2013,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
continue; continue;
} }
sprintf(tbFName, "%s.%s", dbFName, stbName); (void)sprintf(tbFName, "%s.%s", dbFName, stbName);
continue; continue;
} }
@ -1898,7 +2060,7 @@ int32_t ctgChkSetBasicAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res
} }
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&pReq->tbName, dbFName); (void)tNameGetFullDbName(&pReq->tbName, dbFName);
// since that we add read/write previliges when create db, there is no need to check createdDbs // since that we add read/write previliges when create db, there is no need to check createdDbs
#if 0 #if 0
@ -1982,9 +2144,9 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res)
char viewFName[TSDB_VIEW_FNAME_LEN]; char viewFName[TSDB_VIEW_FNAME_LEN];
if (IS_SYS_DBNAME(req->pRawReq->tbName.dbname)) { if (IS_SYS_DBNAME(req->pRawReq->tbName.dbname)) {
snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); (void)snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
} else { } else {
tNameExtractFullName(&req->pRawReq->tbName, viewFName); (void)tNameExtractFullName(&req->pRawReq->tbName, viewFName);
} }
int32_t len = strlen(viewFName) + 1; int32_t len = strlen(viewFName) + 1;
@ -2358,7 +2520,7 @@ void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) {
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
} }
memcpy(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); TAOS_MEMCPY(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache));
} }
void ctgGetGlobalCacheSize(uint64_t *pSize) { void ctgGetGlobalCacheSize(uint64_t *pSize) {
@ -2385,12 +2547,19 @@ int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx) {
int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t dbNum = taosArrayGetSize(pCtx->pNames);
for (int32_t i = 0; i < dbNum; ++i) { for (int32_t i = 0; i < dbNum; ++i) {
STablesReq* pReq = taosArrayGet(pCtx->pNames, i); STablesReq* pReq = taosArrayGet(pCtx->pNames, i);
if (NULL == pReq) {
qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pCtx->pNames));
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t viewNum = taosArrayGetSize(pReq->pTables); int32_t viewNum = taosArrayGetSize(pReq->pTables);
ctgDebug("start to check views in db %s, viewNum %d", pReq->dbFName, viewNum); ctgDebug("start to check views in db %s, viewNum %d", pReq->dbFName, viewNum);
for (int32_t m = 0; m < viewNum; ++m) { for (int32_t m = 0; m < viewNum; ++m) {
taosArrayPush(pCtx->pResList, &(SMetaData){0}); if (NULL == taosArrayPush(pCtx->pResList, &(SMetaData){0})) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
} }
@ -2415,17 +2584,27 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) {
if (pViewMeta->pSchema == NULL) { if (pViewMeta->pSchema == NULL) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
memcpy(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema));
TAOS_MEMCPY(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) { uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) {
if (!pTsmaInfo) return 0; if (!pTsmaInfo) {
return 0;
}
uint64_t size = sizeof(STableTSMAInfo); uint64_t size = sizeof(STableTSMAInfo);
if (pTsmaInfo->pFuncs) size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size; if (pTsmaInfo->pFuncs) {
if (pTsmaInfo->pTags) size += sizeof(SSchema) * pTsmaInfo->pTags->size; size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size;
if (pTsmaInfo->pUsedCols) size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size; }
if (pTsmaInfo->pTags) {
size += sizeof(SSchema) * pTsmaInfo->pTags->size;
}
if (pTsmaInfo->pUsedCols) {
size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size;
}
return size; return size;
} }
@ -2435,8 +2614,14 @@ bool hasOutOfDateTSMACache(SArray* pTsmas) {
} }
for (int32_t i = 0; i < pTsmas->size; ++i) { for (int32_t i = 0; i < pTsmas->size; ++i) {
STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i); STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i);
if (isCtgTSMACacheOutOfDate(pTsmaInfo)) return true; if (NULL == pTsmaInfo) {
ASSERT(0);
}
if (isCtgTSMACacheOutOfDate(pTsmaInfo)) {
return true;
}
} }
return false; return false;
} }
@ -2457,6 +2642,9 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t*
CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName) { CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName) {
if (NULL == (*pFetchs)) { if (NULL == (*pFetchs)) {
*pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgTSMAFetch)); *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgTSMAFetch));
if (NULL == *pFetchs) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
} }
SCtgTSMAFetch fetch = {0}; SCtgTSMAFetch fetch = {0};
@ -2467,9 +2655,13 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t*
fetch.flag = flag; fetch.flag = flag;
fetch.fetchType = fetchType; fetch.fetchType = fetchType;
if (sourceTbName) fetch.tsmaSourceTbName = *sourceTbName; if (sourceTbName) {
fetch.tsmaSourceTbName = *sourceTbName;
}
taosArrayPush(*pFetchs, &fetch); if (NULL == taosArrayPush(*pFetchs, &fetch)) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

22
source/libs/function/src/tudf.c
View File

@ -143,14 +143,29 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
udfdPathLdLib[udfdLdLibPathLen] = ':'; udfdPathLdLib[udfdLdLibPathLen] = ':';
strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1); strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen - 1);
if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) {
fnInfo("udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); fnInfo("[UDFD]udfd LD_LIBRARY_PATH: %s", udfdPathLdLib);
} else { } else {
fnError("can not set correct udfd LD_LIBRARY_PATH"); fnError("[UDFD]can not set correct udfd LD_LIBRARY_PATH");
} }
char ldLibPathEnvItem[1024 + 32] = {0}; char ldLibPathEnvItem[1024 + 32] = {0};
snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", udfdPathLdLib); snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", udfdPathLdLib);
char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem, NULL}; char *taosFqdnEnvItem = NULL;
char *taosFqdn = getenv("TAOS_FQDN");
if (taosFqdn != NULL) {
taosFqdnEnvItem = taosMemoryMalloc(strlen("TAOS_FQDN=") + strlen(taosFqdn) + 1);
if (taosFqdnEnvItem != NULL) {
strcpy(taosFqdnEnvItem, "TAOS_FQDN=");
strcat(taosFqdnEnvItem, taosFqdn);
fnInfo("[UDFD]Succsess to set TAOS_FQDN:%s", taosFqdn);
} else {
fnError("[UDFD]Failed to allocate memory for TAOS_FQDN");
return TSDB_CODE_OUT_OF_MEMORY;
}
}
char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem,taosFqdnEnvItem, NULL};
options.env = envUdfd; options.env = envUdfd;
int err = uv_spawn(&pData->loop, &pData->process, &options); int err = uv_spawn(&pData->loop, &pData->process, &options);
@ -180,6 +195,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
} else { } else {
fnInfo("udfd is initialized"); fnInfo("udfd is initialized");
} }
if(taosFqdnEnvItem) taosMemoryFree(taosFqdnEnvItem);
return err; return err;
} }

5
source/libs/function/src/udfd.c
View File

@ -1421,22 +1421,26 @@ int main(int argc, char *argv[]) {
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error"); fnError("failed to start since read config error");
taosCloseLog();
return -2; return -2;
} }
initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
if (udfdOpenClientRpc() != 0) { if (udfdOpenClientRpc() != 0) {
fnError("open rpc connection to mnode failed"); fnError("open rpc connection to mnode failed");
taosCloseLog();
return -3; return -3;
} }
if (udfdCreateUdfSourceDir() != 0) { if (udfdCreateUdfSourceDir() != 0) {
fnError("create udf source directory failed"); fnError("create udf source directory failed");
taosCloseLog();
return -4; return -4;
} }
if (udfdUvInit() != 0) { if (udfdUvInit() != 0) {
fnError("uv init failure"); fnError("uv init failure");
taosCloseLog();
return -5; return -5;
} }
@ -1452,6 +1456,7 @@ int main(int argc, char *argv[]) {
udfdDeinitScriptPlugins(); udfdDeinitScriptPlugins();
taosCloseLog();
udfdCleanup(); udfdCleanup();
return 0; return 0;
} }

2
source/libs/parser/src/parTranslater.c
View File

@ -13179,7 +13179,7 @@ static int32_t buildTagIndexForBindTags(SMsgBuf* pMsgBuf, SCreateSubTableFromFil
if (code) break; if (code) break;
if (taosHashPut(pIdxHash, &idx, sizeof(idx), NULL, 0) < 0) { if (taosHashPut(pIdxHash, &idx, sizeof(idx), NULL, 0) != 0) {
code = terrno; code = terrno;
goto _OUT; goto _OUT;
} }

76
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -77,10 +77,10 @@ void schtInitLogFile() {
const int32_t maxLogFileNum = 10; const int32_t maxLogFileNum = 10;
tsAsyncLog = 0; tsAsyncLog = 0;
qDebugFlag = 159; qDebugFlag = 159;
strcpy(tsLogDir, TD_LOG_DIR_PATH); TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH);
if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
printf("failed to open log file in directory:%s\n", tsLogDir); (void)printf("failed to open log file in directory:%s\n", tsLogDir);
} }
} }
@ -628,7 +628,7 @@ void schtFreeQueryJob(int32_t freeThread) {
schedulerFreeJob(&job, 0); schedulerFreeJob(&job, 0);
if (freeThread) { if (freeThread) {
if (++freeNum % schtTestPrintNum == 0) { if (++freeNum % schtTestPrintNum == 0) {
printf("FreeNum:%d\n", freeNum); (void)printf("FreeNum:%d\n", freeNum);
} }
} }
} }
@ -667,7 +667,7 @@ void *schtRunJobThread(void *aa) {
SQueryNodeLoad load = {0}; SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1; load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031; load.addr.epSet.eps[0].port = 6031;
if (NULL == taosArrayPush(qnodeList, &load)) { if (NULL == taosArrayPush(qnodeList, &load)) {
assert(0); assert(0);
@ -800,7 +800,7 @@ void *schtRunJobThread(void *aa) {
schtFreeQueryDag(dag); schtFreeQueryDag(dag);
if (++jobFinished % schtTestPrintNum == 0) { if (++jobFinished % schtTestPrintNum == 0) {
printf("jobFinished:%d\n", jobFinished); (void)printf("jobFinished:%d\n", jobFinished);
} }
++schtQueryId; ++schtQueryId;
@ -834,9 +834,9 @@ TEST(queryTest, normalCase) {
SQueryNodeLoad load = {0}; SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1; load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031; load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load); assert(taosArrayPush(qnodeList, &load) != NULL);
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -872,7 +872,7 @@ TEST(queryTest, normalCase) {
SDataBuf msg = {0}; SDataBuf msg = {0};
void *rmsg = NULL; void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg); assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg));
msg.msgType = TDMT_SCH_QUERY_RSP; msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg; msg.pData = rmsg;
@ -888,7 +888,7 @@ TEST(queryTest, normalCase) {
if (JOB_TASK_STATUS_EXEC == task->status) { if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0}; SDataBuf msg = {0};
void *rmsg = NULL; void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg); assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg));
msg.msgType = TDMT_SCH_QUERY_RSP; msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg; msg.pData = rmsg;
@ -909,10 +909,10 @@ TEST(queryTest, normalCase) {
} }
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); assert(0 == taosThreadAttrInit(&thattr));
TdThread thread1; TdThread thread1;
taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job));
void *data = NULL; void *data = NULL;
req.syncReq = true; req.syncReq = true;
@ -926,13 +926,13 @@ TEST(queryTest, normalCase) {
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
schReleaseJob(job); (void)schReleaseJob(job);
schedulerDestroy(); schedulerDestroy();
schedulerFreeJob(&job, 0); schedulerFreeJob(&job, 0);
taosThreadJoin(thread1, NULL); (void)taosThreadJoin(thread1, NULL);
} }
TEST(queryTest, readyFirstCase) { TEST(queryTest, readyFirstCase) {
@ -948,9 +948,9 @@ TEST(queryTest, readyFirstCase) {
SQueryNodeLoad load = {0}; SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1; load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031; load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load); assert(NULL != taosArrayPush(qnodeList, &load));
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -985,7 +985,7 @@ TEST(queryTest, readyFirstCase) {
SDataBuf msg = {0}; SDataBuf msg = {0};
void *rmsg = NULL; void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg); assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg));
msg.msgType = TDMT_SCH_QUERY_RSP; msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg; msg.pData = rmsg;
@ -1002,7 +1002,7 @@ TEST(queryTest, readyFirstCase) {
if (JOB_TASK_STATUS_EXEC == task->status) { if (JOB_TASK_STATUS_EXEC == task->status) {
SDataBuf msg = {0}; SDataBuf msg = {0};
void *rmsg = NULL; void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg); assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg));
msg.msgType = TDMT_SCH_QUERY_RSP; msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg; msg.pData = rmsg;
@ -1023,10 +1023,10 @@ TEST(queryTest, readyFirstCase) {
} }
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); assert(0 == taosThreadAttrInit(&thattr));
TdThread thread1; TdThread thread1;
taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job));
void *data = NULL; void *data = NULL;
req.syncReq = true; req.syncReq = true;
@ -1039,13 +1039,13 @@ TEST(queryTest, readyFirstCase) {
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
schReleaseJob(job); (void)schReleaseJob(job);
schedulerDestroy(); schedulerDestroy();
schedulerFreeJob(&job, 0); schedulerFreeJob(&job, 0);
taosThreadJoin(thread1, NULL); (void)taosThreadJoin(thread1, NULL);
} }
TEST(queryTest, flowCtrlCase) { TEST(queryTest, flowCtrlCase) {
@ -1065,9 +1065,9 @@ TEST(queryTest, flowCtrlCase) {
SQueryNodeLoad load = {0}; SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1; load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031; load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load); assert(NULL != taosArrayPush(qnodeList, &load));
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1078,7 +1078,7 @@ TEST(queryTest, flowCtrlCase) {
schtSetExecNode(); schtSetExecNode();
schtSetAsyncSendMsgToServer(); schtSetAsyncSendMsgToServer();
initTaskQueue(); assert(0 == initTaskQueue());
int32_t queryDone = 0; int32_t queryDone = 0;
SRequestConnInfo conn = {0}; SRequestConnInfo conn = {0};
@ -1106,7 +1106,7 @@ TEST(queryTest, flowCtrlCase) {
if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) { if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) {
SDataBuf msg = {0}; SDataBuf msg = {0};
void *rmsg = NULL; void *rmsg = NULL;
schtBuildQueryRspMsg(&msg.len, &rmsg); assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg));
msg.msgType = TDMT_SCH_QUERY_RSP; msg.msgType = TDMT_SCH_QUERY_RSP;
msg.pData = rmsg; msg.pData = rmsg;
@ -1120,10 +1120,10 @@ TEST(queryTest, flowCtrlCase) {
} }
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); assert(0 == taosThreadAttrInit(&thattr));
TdThread thread1; TdThread thread1;
taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job));
void *data = NULL; void *data = NULL;
req.syncReq = true; req.syncReq = true;
@ -1136,13 +1136,13 @@ TEST(queryTest, flowCtrlCase) {
ASSERT_EQ(pRsp->numOfRows, 10); ASSERT_EQ(pRsp->numOfRows, 10);
taosMemoryFreeClear(data); taosMemoryFreeClear(data);
schReleaseJob(job); (void)schReleaseJob(job);
schedulerDestroy(); schedulerDestroy();
schedulerFreeJob(&job, 0); schedulerFreeJob(&job, 0);
taosThreadJoin(thread1, NULL); (void)taosThreadJoin(thread1, NULL);
} }
TEST(insertTest, normalCase) { TEST(insertTest, normalCase) {
@ -1158,9 +1158,9 @@ TEST(insertTest, normalCase) {
SQueryNodeLoad load = {0}; SQueryNodeLoad load = {0};
load.addr.epSet.numOfEps = 1; load.addr.epSet.numOfEps = 1;
strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep");
load.addr.epSet.eps[0].port = 6031; load.addr.epSet.eps[0].port = 6031;
taosArrayPush(qnodeList, &load); assert(NULL != taosArrayPush(qnodeList, &load));
int32_t code = schedulerInit(); int32_t code = schedulerInit();
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -1171,12 +1171,12 @@ TEST(insertTest, normalCase) {
schtSetAsyncSendMsgToServer(); schtSetAsyncSendMsgToServer();
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); assert(0 == taosThreadAttrInit(&thattr));
schtJobDone = false; schtJobDone = false;
TdThread thread1; TdThread thread1;
taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); assert(0 == taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId));
int32_t queryDone = 0; int32_t queryDone = 0;
SRequestConnInfo conn = {0}; SRequestConnInfo conn = {0};
@ -1204,17 +1204,17 @@ TEST(insertTest, normalCase) {
schedulerDestroy(); schedulerDestroy();
taosThreadJoin(thread1, NULL); (void)taosThreadJoin(thread1, NULL);
} }
TEST(multiThread, forceFree) { TEST(multiThread, forceFree) {
TdThreadAttr thattr; TdThreadAttr thattr;
taosThreadAttrInit(&thattr); assert(0 == taosThreadAttrInit(&thattr));
TdThread thread1, thread2, thread3; TdThread thread1, thread2, thread3;
taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); assert(0 == taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL));
// taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); // taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL);
taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); assert(0 == taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL));
while (true) { while (true) {
if (schtTestDeadLoop) { if (schtTestDeadLoop) {
@ -1231,7 +1231,7 @@ TEST(multiThread, forceFree) {
TEST(otherTest, otherCase) { TEST(otherTest, otherCase) {
// excpet test // excpet test
schReleaseJob(0); (void)schReleaseJob(0);
schFreeRpcCtx(NULL); schFreeRpcCtx(NULL);
char* ep = NULL; char* ep = NULL;

49
source/libs/stream/inc/streamInt.h
View File

@ -65,11 +65,6 @@ struct SActiveCheckpointInfo {
tmr_h pSendReadyMsgTmr; tmr_h pSendReadyMsgTmr;
}; };
struct SConsensusCheckpoint {
int8_t inProcess;
};
typedef struct { typedef struct {
int8_t type; int8_t type;
SSDataBlock* pBlock; SSDataBlock* pBlock;
@ -166,19 +161,19 @@ extern int32_t streamMetaId;
int32_t streamTimerInit(); int32_t streamTimerInit();
void streamTimerCleanUp(); void streamTimerCleanUp();
void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration); void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration);
int32_t streamDispatchStreamBlock(SStreamTask* pTask); int32_t streamDispatchStreamBlock(SStreamTask* pTask);
void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups);
void clearBufferedDispatchMsg(SStreamTask* pTask); void clearBufferedDispatchMsg(SStreamTask* pTask);
int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg,
SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SStreamDataBlock** pBlock);
SArray* pRes); int32_t createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes,
void destroyStreamDataBlock(SStreamDataBlock* pBlock); SStreamDataBlock** pBlock);
void destroyStreamDataBlock(SStreamDataBlock* pBlock);
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData, const char* idstr); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData, const char* idstr);
int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock); int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock);
@ -206,24 +201,24 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIt
int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem); int32_t streamQueueItemGetSize(const SStreamQueueItem* pItem);
void streamQueueItemIncSize(const SStreamQueueItem* pItem, int32_t size); void streamQueueItemIncSize(const SStreamQueueItem* pItem, int32_t size);
const char* streamQueueItemGetTypeStr(int32_t type); const char* streamQueueItemGetTypeStr(int32_t type);
SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes);
int32_t streamTransferStatePrepare(SStreamTask* pTask); int32_t streamTransferStatePrepare(SStreamTask* pTask);
SStreamQueue* streamQueueOpen(int64_t cap); int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ);
void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId);
void streamQueueProcessSuccess(SStreamQueue* queue); void streamQueueProcessSuccess(SStreamQueue* queue);
void streamQueueProcessFail(SStreamQueue* queue); void streamQueueProcessFail(SStreamQueue* queue);
void* streamQueueNextItem(SStreamQueue* pQueue); void* streamQueueNextItem(SStreamQueue* pQueue);
void streamFreeQitem(SStreamQueueItem* data); void streamFreeQitem(SStreamQueueItem* data);
int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue);
void streamMetaRemoveDB(void* arg, char* key); void streamMetaRemoveDB(void* arg, char* key);
void streamMetaHbToMnode(void* param, void* tmrId); void streamMetaHbToMnode(void* param, void* tmrId);
SMetaHbInfo* createMetaHbInfo(int64_t* pRid); int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes);
void* destroyMetaHbInfo(SMetaHbInfo* pInfo); void destroyMetaHbInfo(SMetaHbInfo* pInfo);
void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta); void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta);
void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSendCount); void streamMetaGetHbSendInfo(SMetaHbInfo* pInfo, int64_t* pStartTs, int32_t* pSendCount);
int32_t streamMetaSendHbHelper(SStreamMeta* pMeta); int32_t streamMetaSendHbHelper(SStreamMeta* pMeta);
ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType(); ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType();

36
source/libs/stream/src/streamCheckStatus.c
View File

@ -40,8 +40,11 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
int64_t* oldStage) { int64_t* oldStage) {
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); SStreamUpstreamEpInfo* pInfo = NULL;
ASSERT(pInfo != NULL); int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
*oldStage = pInfo->stage; *oldStage = pInfo->stage;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
@ -63,7 +66,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
id, upstreamTaskId, vgId, stage, pInfo->stage); id, upstreamTaskId, vgId, stage, pInfo->stage);
// record the checkpoint failure id and sent to mnode // record the checkpoint failure id and sent to mnode
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} }
@ -72,7 +75,7 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_
if (pInfo->stage != stage) { if (pInfo->stage != stage) {
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (status == TASK_STATUS__CK) { if (status == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} }
@ -166,15 +169,16 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS
taskId, pReq->upstreamTaskId, pReq->upstreamNodeId, pMeta->vgId); taskId, pReq->upstreamTaskId, pReq->upstreamNodeId, pMeta->vgId);
pRsp->status = TASK_DOWNSTREAM_NOT_LEADER; pRsp->status = TASK_DOWNSTREAM_NOT_LEADER;
} else { } else {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, taskId, &pTask);
if (pTask != NULL) { if (pTask != NULL) {
pRsp->status = pRsp->status =
streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64
") task:0x%x (vgId:%d), check_status:%d", ") task:0x%x (vgId:%d), check_status:%d",
pTask->id.idStr, pState->name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pTask->id.idStr, pState.name, pRsp->oldStage, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId,
pRsp->status); pRsp->status);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} else { } else {
@ -647,12 +651,12 @@ int32_t addDownstreamFailedStatusResultAsync(SMsgCb* pMsgCb, int32_t vgId, int64
void rspMonitorFn(void* param, void* tmrId) { void rspMonitorFn(void* param, void* tmrId) {
SStreamTask* pTask = param; SStreamTask* pTask = param;
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
SStreamTaskState* pStat = streamTaskGetStatus(pTask); SStreamTaskState pStat = streamTaskGetStatus(pTask);
STaskCheckInfo* pInfo = &pTask->taskCheckInfo; STaskCheckInfo* pInfo = &pTask->taskCheckInfo;
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int64_t timeoutDuration = now - pInfo->timeoutStartTs; int64_t timeoutDuration = now - pInfo->timeoutStartTs;
ETaskStatus state = pStat->state; ETaskStatus state = pStat.state;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t numOfReady = 0; int32_t numOfReady = 0;
int32_t numOfFault = 0; int32_t numOfFault = 0;
@ -665,7 +669,7 @@ void rspMonitorFn(void* param, void* tmrId) {
if (state == TASK_STATUS__STOP) { if (state == TASK_STATUS__STOP) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat.name, vgId, ref);
streamTaskCompleteCheckRsp(pInfo, true, id); streamTaskCompleteCheckRsp(pInfo, true, id);
@ -681,7 +685,7 @@ void rspMonitorFn(void* param, void* tmrId) {
if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) { if (state == TASK_STATUS__DROPPING || state == TASK_STATUS__READY) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat->name, vgId, ref); stDebug("s-task:%s status:%s vgId:%d quit from monitor check-rsp tmr, ref:%d", id, pStat.name, vgId, ref);
streamTaskCompleteCheckRsp(pInfo, true, id); streamTaskCompleteCheckRsp(pInfo, true, id);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -691,7 +695,7 @@ void rspMonitorFn(void* param, void* tmrId) {
taosThreadMutexLock(&pInfo->checkInfoLock); taosThreadMutexLock(&pInfo->checkInfoLock);
if (pInfo->notReadyTasks == 0) { if (pInfo->notReadyTasks == 0) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat->name, stDebug("s-task:%s status:%s vgId:%d all downstream ready, quit from monitor rsp tmr, ref:%d", id, pStat.name,
vgId, ref); vgId, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
@ -703,7 +707,7 @@ void rspMonitorFn(void* param, void* tmrId) {
SArray* pNotReadyList = taosArrayInit(4, sizeof(int64_t)); SArray* pNotReadyList = taosArrayInit(4, sizeof(int64_t));
SArray* pTimeoutList = taosArrayInit(4, sizeof(int64_t)); SArray* pTimeoutList = taosArrayInit(4, sizeof(int64_t));
if (pStat->state == TASK_STATUS__UNINIT) { if (pStat.state == TASK_STATUS__UNINIT) {
getCheckRspStatus(pInfo, timeoutDuration, &numOfReady, &numOfFault, &numOfNotRsp, pTimeoutList, pNotReadyList, id); getCheckRspStatus(pInfo, timeoutDuration, &numOfReady, &numOfFault, &numOfNotRsp, pTimeoutList, pNotReadyList, id);
numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList); numOfNotReady = (int32_t)taosArrayGetSize(pNotReadyList);
@ -716,7 +720,7 @@ void rspMonitorFn(void* param, void* tmrId) {
stDebug( stDebug(
"s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart " "s-task:%s status:%s vgId:%d all rsp. quit from monitor rsp tmr, since vnode-transfer/leader-change/restart "
"detected, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d", "detected, total:%d, notRsp:%d, notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
id, pStat->name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); taosThreadMutexUnlock(&pInfo->checkInfoLock);
@ -727,7 +731,7 @@ void rspMonitorFn(void* param, void* tmrId) {
return; return;
} }
} else { // unexpected status } else { // unexpected status
stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat->name); stError("s-task:%s unexpected task status:%s during waiting for check rsp", id, pStat.name);
} }
// checking of downstream tasks has been stopped by other threads // checking of downstream tasks has been stopped by other threads
@ -736,7 +740,7 @@ void rspMonitorFn(void* param, void* tmrId) {
stDebug( stDebug(
"s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, total:%d, notRsp:%d, " "s-task:%s status:%s vgId:%d stopped by other threads to check downstream process, total:%d, notRsp:%d, "
"notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d", "notReady:%d, fault:%d, timeout:%d, ready:%d ref:%d",
id, pStat->name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref); id, pStat.name, vgId, total, numOfNotRsp, numOfNotReady, numOfFault, numOfTimeout, numOfReady, ref);
streamTaskCompleteCheckRsp(pInfo, false, id); streamTaskCompleteCheckRsp(pInfo, false, id);
taosThreadMutexUnlock(&pInfo->checkInfoLock); taosThreadMutexUnlock(&pInfo->checkInfoLock);

231
source/libs/stream/src/streamCheckpoint.c
View File

@ -29,14 +29,12 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint
static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList);
static void checkpointTriggerMonitorFn(void* param, void* tmrId); static void checkpointTriggerMonitorFn(void* param, void* tmrId);
SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId,
int32_t transId, int32_t srcTaskId) { int32_t srcTaskId, SStreamDataBlock** pRes) {
SStreamDataBlock* pChkpoint; SStreamDataBlock* pChkpoint = NULL;
int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pChkpoint); int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock), (void**)&pChkpoint);
if (code) { if (code) {
terrno = code; return code;
return NULL;
} }
pChkpoint->type = checkpointType; pChkpoint->type = checkpointType;
@ -48,8 +46,7 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) { if (pBlock == NULL) {
taosFreeQitem(pChkpoint); taosFreeQitem(pChkpoint);
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pBlock->info.type = STREAM_CHECKPOINT; pBlock->info.type = STREAM_CHECKPOINT;
@ -62,28 +59,37 @@ SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpoint
if (pChkpoint->blocks == NULL) { if (pChkpoint->blocks == NULL) {
taosMemoryFree(pBlock); taosMemoryFree(pBlock);
taosFreeQitem(pChkpoint); taosFreeQitem(pChkpoint);
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
taosArrayPush(pChkpoint->blocks, pBlock); void* p = taosArrayPush(pChkpoint->blocks, pBlock);
if (p == NULL) {
taosArrayDestroy(pChkpoint->blocks);
taosMemoryFree(pBlock);
taosFreeQitem(pChkpoint);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pRes = pChkpoint;
taosMemoryFree(pBlock); taosMemoryFree(pBlock);
terrno = 0; return TSDB_CODE_SUCCESS;
return pChkpoint;
} }
// this message must be put into inputq successfully, continue retrying until it succeeds
int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId, int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId,
int32_t srcTaskId) { int32_t srcTaskId) {
SStreamDataBlock* pCheckpoint = createChkptTriggerBlock(pTask, checkpointType, checkpointId, transId, srcTaskId); SStreamDataBlock* pCheckpoint = NULL;
int32_t code = createChkptTriggerBlock(pTask, checkpointType, checkpointId, transId, srcTaskId, &pCheckpoint);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pCheckpoint) < 0) { if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pCheckpoint) < 0) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
streamTrySchedExec(pTask); return streamTrySchedExec(pTask);
return TSDB_CODE_SUCCESS;
} }
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) { int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) {
@ -178,22 +184,26 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
if (pTask->chkInfo.checkpointId > checkpointId) { if (pTask->chkInfo.checkpointId > checkpointId) {
stError("s-task:%s vgId:%d current checkpointId:%" PRId64 stError("s-task:%s vgId:%d current checkpointId:%" PRId64
" recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard",
id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId);
taosThreadMutexUnlock(&pTask->lock); code = taosThreadMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return TSDB_CODE_SUCCESS; return code;
} }
if (pTask->chkInfo.checkpointId == checkpointId) { if (pTask->chkInfo.checkpointId == checkpointId) {
{ // send checkpoint-ready msg to upstream { // send checkpoint-ready msg to upstream
SRpcMsg msg = {0}; SRpcMsg msg = {0};
SStreamUpstreamEpInfo* pInfo = NULL;
code = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId);
initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg);
tmsgSendReq(&pInfo->epSet, &msg); tmsgSendReq(&pInfo->epSet, &msg);
} }
@ -204,30 +214,30 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
id, vgId, pBlock->srcTaskId); id, vgId, pBlock->srcTaskId);
streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId);
taosThreadMutexUnlock(&pTask->lock); code = taosThreadMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return TSDB_CODE_SUCCESS; return code;
} }
if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) {
if (pActiveInfo->activeId != checkpointId) { if (pActiveInfo->activeId != checkpointId) {
stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64
" discard", " discard",
id, vgId, pActiveInfo->activeId, checkpointId); id, vgId, pActiveInfo->activeId, checkpointId);
taosThreadMutexUnlock(&pTask->lock); code = taosThreadMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return TSDB_CODE_SUCCESS; return code;
} else { // checkpointId == pActiveInfo->activeId } else { // checkpointId == pActiveInfo->activeId
if (pActiveInfo->allUpstreamTriggerRecv == 1) { if (pActiveInfo->allUpstreamTriggerRecv == 1) {
stDebug( stDebug(
"s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, "
"checkpointId:%" PRId64 " transId:%d", "checkpointId:%" PRId64 " transId:%d",
id, vgId, checkpointId, transId); id, vgId, checkpointId, transId);
taosThreadMutexUnlock(&pTask->lock); code = taosThreadMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return TSDB_CODE_SUCCESS; return code;
} }
if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@ -240,23 +250,23 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
", prev recvTs:%" PRId64 " discard", ", prev recvTs:%" PRId64 " discard",
pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs);
taosThreadMutexUnlock(&pTask->lock); code = taosThreadMutexUnlock(&pTask->lock);
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
return TSDB_CODE_SUCCESS; return code;
} }
} }
} }
} }
} }
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64
", transId:%d current active checkpointId:%" PRId64, ", transId:%d current active checkpointId:%" PRId64,
id, vgId, pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, transId, checkpointId); id, vgId, pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, transId, checkpointId);
// set task status // set task status
if (streamTaskGetStatus(pTask)->state != TASK_STATUS__CK) { if (streamTaskGetStatus(pTask).state != TASK_STATUS__CK) {
pActiveInfo->activeId = checkpointId; pActiveInfo->activeId = checkpointId;
pActiveInfo->transId = transId; pActiveInfo->transId = transId;
@ -345,9 +355,9 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
ASSERT(total > 0); ASSERT(total > 0);
// 1. not in checkpoint status now // 1. not in checkpoint status now
SStreamTaskState* pStat = streamTaskGetStatus(pTask); SStreamTaskState pStat = streamTaskGetStatus(pTask);
if (pStat->state != TASK_STATUS__CK) { if (pStat.state != TASK_STATUS__CK) {
stError("s-task:%s status:%s discard checkpoint-ready msg from task:0x%x", id, pStat->name, downstreamTaskId); stError("s-task:%s status:%s discard checkpoint-ready msg from task:0x%x", id, pStat.name, downstreamTaskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
@ -355,11 +365,11 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
if (pTask->chkInfo.checkpointId > checkpointId || pInfo->activeId != checkpointId) { if (pTask->chkInfo.checkpointId > checkpointId || pInfo->activeId != checkpointId) {
stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64 stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64
") from task:0x%x, expired and discard ", ") from task:0x%x, expired and discard ",
id, pStat->name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId); id, pStat.name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId);
return -1; return -1;
} }
taosThreadMutexLock(&pInfo->lock); (void) taosThreadMutexLock(&pInfo->lock);
// only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task
int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
@ -386,7 +396,7 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList); int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
int32_t transId = pInfo->transId; int32_t transId = pInfo->transId;
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
if (notReady == 0) { if (notReady == 0) {
stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id); stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id);
@ -401,7 +411,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
int32_t numOfConfirmed = 0; int32_t numOfConfirmed = 0;
taosThreadMutexLock(&pInfo->lock); (void) taosThreadMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) {
STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i);
if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) {
@ -422,7 +432,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream
stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr,
numOfConfirmed, checkpointId); numOfConfirmed, checkpointId);
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -430,12 +440,12 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->chkInfo.startTs = 0; // clear the recorded start time
streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks
taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); (void) taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock);
streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo); streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo);
if (clearChkpReadyMsg) { if (clearChkpReadyMsg) {
streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo); streamClearChkptReadyMsg(pTask->chkInfo.pActiveInfo);
} }
taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); (void) taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock);
} }
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) { int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq) {
@ -445,7 +455,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SCheckpointInfo* pInfo = &pTask->chkInfo; SCheckpointInfo* pInfo = &pTask->chkInfo;
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
if (pReq->checkpointId <= pInfo->checkpointId) { if (pReq->checkpointId <= pInfo->checkpointId) {
stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64
@ -453,13 +463,13 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
" transId:%d ignored", " transId:%d ignored",
id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer,
pReq->transId); pReq->transId);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
{ // destroy the related fill-history tasks { // destroy the related fill-history tasks
// drop task should not in the meta-lock, and drop the related fill-history task now // drop task should not in the meta-lock, and drop the related fill-history task now
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
if (pReq->dropRelHTask) { if (pReq->dropRelHTask) {
streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); (void) streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d", stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d",
id, vgId, pReq->taskId, numOfTasks); id, vgId, pReq->taskId, numOfTasks);
@ -473,17 +483,17 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamTaskState* pStatus = streamTaskGetStatus(pTask); SStreamTaskState pStatus = streamTaskGetStatus(pTask);
if (!restored) { // during restore procedure, do update checkpoint-info if (!restored) { // during restore procedure, do update checkpoint-info
stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64 stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64
" checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64, " checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64,
id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, id, vgId, pStatus.name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer,
pInfo->checkpointTime, pReq->checkpointTs); pInfo->checkpointTime, pReq->checkpointTs);
} else { // not in restore status, must be in checkpoint status } else { // not in restore status, must be in checkpoint status
stDebug("s-task:%s vgId:%d status:%s start to update the checkpoint-info, checkpointId:%" PRId64 "->%" PRId64 stDebug("s-task:%s vgId:%d status:%s start to update the checkpoint-info, checkpointId:%" PRId64 "->%" PRId64
" checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64, " checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64,
id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, id, vgId, pStatus.name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer,
pInfo->checkpointTime, pReq->checkpointTs); pInfo->checkpointTime, pReq->checkpointTs);
} }
@ -496,11 +506,11 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
streamTaskClearCheckInfo(pTask, true); streamTaskClearCheckInfo(pTask, true);
if (pStatus->state == TASK_STATUS__CK) { if (pStatus.state == TASK_STATUS__CK) {
// todo handle error // todo handle error
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
} else { } else {
stDebug("s-task:0x%x vgId:%d not handle checkpoint-done event, status:%s", pReq->taskId, vgId, pStatus->name); stDebug("s-task:0x%x vgId:%d not handle checkpoint-done event, status:%s", pReq->taskId, vgId, pStatus.name);
} }
if (pReq->dropRelHTask) { if (pReq->dropRelHTask) {
@ -510,7 +520,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
} }
stDebug("s-task:0x%x set the persistent status attr to be ready, prev:%s, status in sm:%s", pReq->taskId, stDebug("s-task:0x%x set the persistent status attr to be ready, prev:%s, status in sm:%s", pReq->taskId,
streamTaskGetStatusStr(pTask->status.taskStatus), streamTaskGetStatus(pTask)->name); streamTaskGetStatusStr(pTask->status.taskStatus), streamTaskGetStatus(pTask).name);
pTask->status.taskStatus = TASK_STATUS__READY; pTask->status.taskStatus = TASK_STATUS__READY;
@ -521,12 +531,12 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
return code; return code;
} }
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// drop task should not in the meta-lock, and drop the related fill-history task now // drop task should not in the meta-lock, and drop the related fill-history task now
if (pReq->dropRelHTask) { if (pReq->dropRelHTask) {
streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); (void) streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId);
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId,
(int32_t)pReq->hTaskId, numOfTasks); (int32_t)pReq->hTaskId, numOfTasks);
@ -720,9 +730,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask);
} }
} else { // clear the checkpoint info if failed } else { // clear the checkpoint info if failed
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info streamTaskSetFailedCheckpointId(pTask); // set failed checkpoint id before clear the checkpoint info
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE);
stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, id, ckId);
@ -760,13 +770,13 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
pActiveInfo->checkCounter = 0; pActiveInfo->checkCounter = 0;
stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now); stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, id, vgId, now);
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__CK) { if (pState.state != TASK_STATUS__CK) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
@ -777,14 +787,14 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId, stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", id, vgId,
ref); ref);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
} }
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
taosThreadMutexLock(&pActiveInfo->lock); (void) taosThreadMutexLock(&pActiveInfo->lock);
// send msg to retrieve checkpoint trigger msg // send msg to retrieve checkpoint trigger msg
SArray* pList = pTask->upstreamInfo.pList; SArray* pList = pTask->upstreamInfo.pList;
@ -817,7 +827,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
// do send retrieve checkpoint trigger msg to upstream // do send retrieve checkpoint trigger msg to upstream
int32_t size = taosArrayGetSize(pNotSendList); int32_t size = taosArrayGetSize(pNotSendList);
doSendRetrieveTriggerMsg(pTask, pNotSendList); doSendRetrieveTriggerMsg(pTask, pNotSendList);
taosThreadMutexUnlock(&pActiveInfo->lock); (void) taosThreadMutexUnlock(&pActiveInfo->lock);
// check every 100ms // check every 100ms
if (size > 0) { if (size > 0) {
@ -881,15 +891,15 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
SStreamTaskState* pStatus = streamTaskGetStatus(pTask); SStreamTaskState pStatus = streamTaskGetStatus(pTask);
if (pStatus->state != TASK_STATUS__CK) { if (pStatus.state != TASK_STATUS__CK) {
return false; return false;
} }
taosThreadMutexLock(&pInfo->lock); (void) taosThreadMutexLock(&pInfo->lock);
if (!pInfo->dispatchTrigger) { if (!pInfo->dispatchTrigger) {
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
return false; return false;
} }
@ -911,7 +921,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId)
id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId);
} }
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
return true; return true;
} }
@ -930,11 +940,12 @@ void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_
} }
// record the dispatch checkpoint trigger info in the list // record the dispatch checkpoint trigger info in the list
// memory insufficient may cause the stream computing stopped
void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
taosThreadMutexLock(&pInfo->lock); (void) taosThreadMutexLock(&pInfo->lock);
// outputQ should be empty here // outputQ should be empty here
ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0);
@ -944,31 +955,37 @@ void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) {
STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher;
STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId};
taosArrayPush(pInfo->pDispatchTriggerList, &p); void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
if (px == NULL) {
// pause the stream task, if memory not enough
}
} else { } else {
for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) {
SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i); SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i);
STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId};
taosArrayPush(pInfo->pDispatchTriggerList, &p); void* px = taosArrayPush(pInfo->pDispatchTriggerList, &p);
if (px == NULL) {
// pause the stream task, if memory not enough
}
} }
} }
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
} }
int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int32_t num = 0; int32_t num = 0;
taosThreadMutexLock(&pInfo->lock); (void) taosThreadMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
if (p->recved) { if (p->recved) {
num++; num++;
} }
} }
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
return num; return num;
} }
@ -976,7 +993,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
int32_t taskId = 0; int32_t taskId = 0;
taosThreadMutexLock(&pInfo->lock); (void) taosThreadMutexLock(&pInfo->lock);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
@ -990,7 +1007,7 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
} }
} }
taosThreadMutexUnlock(&pInfo->lock); (void) taosThreadMutexUnlock(&pInfo->lock);
int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask);
int32_t total = streamTaskGetNumOfDownstream(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask);
@ -1047,7 +1064,8 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) {
stDebug("[s3] upload checkpoint:%s", filename); stDebug("[s3] upload checkpoint:%s", filename);
} }
} }
taosCloseDir(&pDir);
(void) taosCloseDir(&pDir);
return code; return code;
} }
@ -1164,78 +1182,37 @@ int32_t deleteCheckpointFile(const char* id, const char* name) {
} }
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) { int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask) {
int32_t code; const char* id = pTask->id.idStr;
int32_t tlen = 0;
int32_t vgId = pTask->pMeta->vgId;
const char* id = pTask->id.idStr;
SCheckpointInfo* pInfo = &pTask->chkInfo;
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
if (pTask->status.sendConsensusChkptId == true) { if (pTask->status.sendConsensusChkptId == true) {
stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id); stDebug("s-task:%s already start to consensus-checkpointId, not start again before it completed", id);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
pTask->status.sendConsensusChkptId = true; pTask->status.sendConsensusChkptId = true;
} }
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
ASSERT(pTask->pBackend == NULL); ASSERT(pTask->pBackend == NULL);
pTask->status.requireConsensusChkptId = true; pTask->status.requireConsensusChkptId = true;
#if 0
SRestoreCheckpointInfo req = {
.streamId = pTask->id.streamId,
.taskId = pTask->id.taskId,
.nodeId = vgId,
.checkpointId = pInfo->checkpointId,
.startTs = pTask->execInfo.created,
};
tEncodeSize(tEncodeRestoreCheckpointInfo, &req, tlen, code);
if (code < 0) {
stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id failed, code:%s", id, vgId, tstrerror(code));
return TSDB_CODE_INVALID_MSG;
}
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id msg failed, code:%s", id, vgId,
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return TSDB_CODE_OUT_OF_MEMORY;
}
SEncoder encoder;
tEncoderInit(&encoder, buf, tlen);
if ((code = tEncodeRestoreCheckpointInfo(&encoder, &req)) < 0) {
rpcFreeCont(buf);
stError("s-task:%s vgId:%d encode stream task latest-checkpoint-id msg failed, code:%s", id, vgId, tstrerror(code));
return -1;
}
tEncoderClear(&encoder);
SRpcMsg msg = {0};
initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CONSEN_CHKPT, buf, tlen);
stDebug("s-task:%s vgId:%d send latest checkpointId:%" PRId64 " to mnode to get the consensus checkpointId", id, vgId,
pInfo->checkpointId);
tmsgSendReq(&pTask->info.mnodeEpset, &msg);
#endif
return 0; return 0;
} }
int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask) { int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask) {
int32_t code = 0; int32_t code = 0;
if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
return code; return code;
} }
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p->state == TASK_STATUS__CK) { if (p.state == TASK_STATUS__CK) {
code = streamTaskSendCheckpointSourceRsp(pTask); code = streamTaskSendCheckpointSourceRsp(pTask);
} }
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
return code; return code;
} }

69
source/libs/stream/src/streamData.c
View File

@ -15,13 +15,11 @@
#include "streamInt.h" #include "streamInt.h"
SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg) { int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg, SStreamDataBlock** pRes) {
SStreamDataBlock* pData; SStreamDataBlock* pData = NULL;
int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen, (void**)&pData); int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, pReq->totalLen, (void**)&pData);
if (code) { if (code) {
terrno = code; return terrno = code;
return NULL;
} }
pData->type = blockType; pData->type = blockType;
@ -32,7 +30,7 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe
SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum); SArray* pArray = taosArrayInit_s(sizeof(SSDataBlock), blockNum);
if (pArray == NULL) { if (pArray == NULL) {
taosFreeQitem(pData); taosFreeQitem(pData);
return NULL; return code;
} }
ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen))); ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen)));
@ -69,37 +67,36 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe
} }
pData->blocks = pArray; pData->blocks = pArray;
return pData; *pRes = pData;
return code;
} }
SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, int32_t createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes,
SArray* pRes) { SStreamDataBlock** pBlock) {
SStreamDataBlock* pStreamBlocks; int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize, (void**)pBlock);
int32_t code = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, resultSize, (void**)&pStreamBlocks);
if (code) { if (code) {
taosArrayClearEx(pRes, (FDelete)blockDataFreeRes); taosArrayClearEx(pRes, (FDelete)blockDataFreeRes);
terrno = code; return terrno = code;
return NULL;
} }
pStreamBlocks->srcTaskId = pTask->id.taskId; (*pBlock)->srcTaskId = pTask->id.taskId;
pStreamBlocks->type = STREAM_INPUT__DATA_BLOCK; (*pBlock)->type = STREAM_INPUT__DATA_BLOCK;
pStreamBlocks->blocks = pRes; (*pBlock)->blocks = pRes;
if (pItem == NULL) { if (pItem == NULL) {
return pStreamBlocks; return code;
} }
if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pItem; SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)pItem;
pStreamBlocks->sourceVer = pSubmit->ver; (*pBlock)->sourceVer = pSubmit->ver;
} else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) {
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pItem; SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)pItem;
pStreamBlocks->sourceVer = pMerged->ver; (*pBlock)->sourceVer = pMerged->ver;
} }
return pStreamBlocks; return code;
} }
void destroyStreamDataBlock(SStreamDataBlock* pBlock) { void destroyStreamDataBlock(SStreamDataBlock* pBlock) {
@ -138,19 +135,19 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type) { int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit) {
SStreamDataSubmit* pDataSubmit; SStreamDataSubmit* pDataSubmit = NULL;
int32_t code = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, pData->msgLen, (void**)&pDataSubmit); int32_t code = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, pData->msgLen, (void**)&pDataSubmit);
if (code) { if (code) {
terrno = code; return code;
return NULL;
} }
pDataSubmit->ver = pData->ver; pDataSubmit->ver = pData->ver;
pDataSubmit->submit = *pData; pDataSubmit->submit = *pData;
pDataSubmit->type = type; pDataSubmit->type = type;
return pDataSubmit; *pSubmit = pDataSubmit;
return TSDB_CODE_SUCCESS;
} }
void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) {
@ -188,8 +185,8 @@ int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubm
} }
// todo handle memory error // todo handle memory error
SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { int32_t streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem, SStreamQueueItem** pRes) {
terrno = 0; *pRes = NULL;
if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) {
SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; SStreamDataBlock* pBlock = (SStreamDataBlock*)dst;
@ -199,7 +196,8 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI
streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem));
taosFreeQitem(pElem); taosFreeQitem(pElem);
return dst; *pRes = dst;
return TSDB_CODE_SUCCESS;
} else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst;
SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem; SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)pElem;
@ -207,12 +205,13 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI
streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem)); streamQueueItemIncSize(dst, streamQueueItemGetSize(pElem));
taosFreeQitem(pElem); taosFreeQitem(pElem);
return dst; *pRes = dst;
*pRes = dst;
return TSDB_CODE_SUCCESS;
} else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) {
SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); SStreamMergedSubmit* pMerged = streamMergedSubmitNew();
if (pMerged == NULL) { if (pMerged == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem)); streamQueueItemIncSize((SStreamQueueItem*)pMerged, streamQueueItemGetSize(pElem));
@ -222,11 +221,13 @@ SStreamQueueItem* streamQueueMergeQueueItem(SStreamQueueItem* dst, SStreamQueueI
taosFreeQitem(dst); taosFreeQitem(dst);
taosFreeQitem(pElem); taosFreeQitem(pElem);
return (SStreamQueueItem*)pMerged;
*pRes = (SStreamQueueItem*)pMerged;
return TSDB_CODE_SUCCESS;
} else { } else {
stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type), stDebug("block type:%s not merged with existed blocks list, type:%d", streamQueueItemGetTypeStr(pElem->type),
dst->type); dst->type);
return NULL; return TSDB_CODE_FAILED;
} }
} }

34
source/libs/stream/src/streamDispatch.c
View File

@ -799,11 +799,11 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) {
stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id); stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id);
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__CK) { if (pState.state != TASK_STATUS__CK) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId, stDebug("s-task:%s vgId:%d status:%s not in checkpoint, quit from monitor checkpoint-ready send, ref:%d", id, vgId,
pState->name, ref); pState.name, ref);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
@ -1118,7 +1118,11 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
STaskCheckpointReadyInfo info = {0}; STaskCheckpointReadyInfo info = {0};
initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId);
@ -1313,7 +1317,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
if (delayDispatch) { if (delayDispatch) {
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
// we only set the dispatch msg info for current checkpoint trans // we only set the dispatch msg info for current checkpoint trans
if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK &&
pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) { pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) {
ASSERT(pTask->chkInfo.pActiveInfo->transId == pMsgInfo->transId); ASSERT(pTask->chkInfo.pActiveInfo->transId == pMsgInfo->transId);
stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed", stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed",
@ -1404,8 +1408,10 @@ static int32_t buildDispatchRsp(const SStreamTask* pTask, const SStreamDispatchR
static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq) { static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq) {
int8_t status = 0; int8_t status = 0;
SStreamDataBlock* pBlock = createStreamBlockFromDispatchMsg(pReq, pReq->type, pReq->srcVgId); SStreamDataBlock* pBlock = NULL;
if (pBlock == NULL) {
int32_t code = createStreamBlockFromDispatchMsg(pReq, pReq->type, pReq->srcVgId, &pBlock);
if (code) {
streamTaskInputFail(pTask); streamTaskInputFail(pTask);
status = TASK_INPUT_STATUS__FAILED; status = TASK_INPUT_STATUS__FAILED;
stError("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, stError("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId,
@ -1415,7 +1421,7 @@ static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDisp
pTask->status.appendTranstateBlock = true; pTask->status.appendTranstateBlock = true;
} }
int32_t code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pBlock); code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pBlock);
// input queue is full, upstream is blocked now // input queue is full, upstream is blocked now
status = (code == TSDB_CODE_SUCCESS) ? TASK_INPUT_STATUS__NORMAL : TASK_INPUT_STATUS__BLOCKED; status = (code == TSDB_CODE_SUCCESS) ? TASK_INPUT_STATUS__NORMAL : TASK_INPUT_STATUS__BLOCKED;
} }
@ -1431,8 +1437,11 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id, stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id,
pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId);
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); SStreamUpstreamEpInfo* pInfo = NULL;
ASSERT(pInfo != NULL); int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pMeta->role == NODE_ROLE_FOLLOWER) { if (pMeta->role == NODE_ROLE_FOLLOWER) {
stError("s-task:%s task on follower received dispatch msgs, dispatch msg rejected", id); stError("s-task:%s task on follower received dispatch msgs, dispatch msg rejected", id);
@ -1465,10 +1474,9 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
{ {
// do send response with the input status // do send response with the input status
int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code)); stError("s-task:%s failed to build dispatch rsp, msgId:%d, code:%s", id, pReq->msgId, tstrerror(code));
terrno = code;
return code; return code;
} }
@ -1477,5 +1485,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
} }
streamTrySchedExec(pTask); streamTrySchedExec(pTask);
return 0; return code;
} }

66
source/libs/stream/src/streamExec.c
View File

@ -27,12 +27,12 @@ static int32_t streamTransferStateDoPrepare(SStreamTask* pTask);
static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks);
bool streamTaskShouldStop(const SStreamTask* pTask) { bool streamTaskShouldStop(const SStreamTask* pTask) {
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
return (pState->state == TASK_STATUS__STOP) || (pState->state == TASK_STATUS__DROPPING); return (pState.state == TASK_STATUS__STOP) || (pState.state == TASK_STATUS__DROPPING);
} }
bool streamTaskShouldPause(const SStreamTask* pTask) { bool streamTaskShouldPause(const SStreamTask* pTask) {
return (streamTaskGetStatus(pTask)->state == TASK_STATUS__PAUSE); return (streamTaskGetStatus(pTask).state == TASK_STATUS__PAUSE);
} }
static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) { static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBlock) {
@ -67,8 +67,10 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); SStreamDataBlock* pStreamBlocks = NULL;
if (pStreamBlocks == NULL) {
int32_t code = createStreamBlockFromResults(pItem, pTask, size, pRes, &pStreamBlocks);
if (code) {
stError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); stError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno));
taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -77,7 +79,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray*
stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, stDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks,
SIZE_IN_MiB(size)); SIZE_IN_MiB(size));
int32_t code = doOutputResultBlockImpl(pTask, pStreamBlocks); code = doOutputResultBlockImpl(pTask, pStreamBlocks);
if (code != TSDB_CODE_SUCCESS) { // back pressure and record position if (code != TSDB_CODE_SUCCESS) { // back pressure and record position
return code; return code;
} }
@ -187,7 +189,12 @@ int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t*
static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (taosArrayGetSize(pRes) > 0) { if (taosArrayGetSize(pRes) > 0) {
SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes); SStreamDataBlock* pStreamBlocks = NULL;
code = createStreamBlockFromResults(NULL, pTask, size, pRes, &pStreamBlocks);
if (code) {
return code;
}
code = doOutputResultBlockImpl(pTask, pStreamBlocks); code = doOutputResultBlockImpl(pTask, pStreamBlocks);
if (code != TSDB_CODE_SUCCESS) { // should not have error code if (code != TSDB_CODE_SUCCESS) { // should not have error code
stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code));
@ -314,7 +321,8 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); SStreamTask* pStreamTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask == NULL) { if (pStreamTask == NULL) {
stError( stError(
"s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related " "s-task:%s failed to find related stream task:0x%x, it may have been destroyed or closed, destroy the related "
@ -336,10 +344,10 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
stDebug( stDebug(
"s-task:%s fill-history task end, status:%s, scan wal elapsed time:%.2fSec, update related stream task:%s " "s-task:%s fill-history task end, status:%s, scan wal elapsed time:%.2fSec, update related stream task:%s "
"info, prepare transfer exec state", "info, prepare transfer exec state",
id, streamTaskGetStatus(pTask)->name, el, pStreamTask->id.idStr); id, streamTaskGetStatus(pTask).name, el, pStreamTask->id.idStr);
} }
ETaskStatus status = streamTaskGetStatus(pStreamTask)->state; ETaskStatus status = streamTaskGetStatus(pStreamTask).state;
STimeWindow* pTimeWindow = &pStreamTask->dataRange.window; STimeWindow* pTimeWindow = &pStreamTask->dataRange.window;
// It must be halted for a source stream task, since when the related scan-history-data task start scan the history // It must be halted for a source stream task, since when the related scan-history-data task start scan the history
@ -349,7 +357,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
} else { } else {
ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING || ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING ||
status == TASK_STATUS__STOP); status == TASK_STATUS__STOP);
int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id, stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id,
pStreamTask->id.idStr, tstrerror(code)); pStreamTask->id.idStr, tstrerror(code));
@ -363,9 +371,9 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
// In case of sink tasks, no need to halt them. // In case of sink tasks, no need to halt them.
// In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to // In case of source tasks and agg tasks, we should HALT them, and wait for them to be idle. And then, it's safe to
// start the task state transfer procedure. // start the task state transfer procedure.
SStreamTaskState* pState = streamTaskGetStatus(pStreamTask); SStreamTaskState pState = streamTaskGetStatus(pStreamTask);
status = pState->state; status = pState.state;
char* p = pState->name; char* p = pState.name;
if (status == TASK_STATUS__STOP || status == TASK_STATUS__DROPPING) { if (status == TASK_STATUS__STOP || status == TASK_STATUS__DROPPING) {
stError("s-task:%s failed to transfer state from fill-history task:%s, status:%s", id, pStreamTask->id.idStr, p); stError("s-task:%s failed to transfer state from fill-history task:%s, status:%s", id, pStreamTask->id.idStr, p);
streamMetaReleaseTask(pMeta, pStreamTask); streamMetaReleaseTask(pMeta, pStreamTask);
@ -390,7 +398,7 @@ int32_t streamTransferStateDoPrepare(SStreamTask* pTask) {
streamTaskSendCheckpointReq(pStreamTask); streamTaskSendCheckpointReq(pStreamTask);
// 3. assign the status to the value that will be kept in disk // 3. assign the status to the value that will be kept in disk
pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask)->state; pStreamTask->status.taskStatus = streamTaskGetStatus(pStreamTask).state;
// 4. open the inputQ for all upstream tasks // 4. open the inputQ for all upstream tasks
streamTaskOpenAllUpstreamInput(pStreamTask); streamTaskOpenAllUpstreamInput(pStreamTask);
@ -416,7 +424,8 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) {
code = streamTransferStateDoPrepare(pTask); code = streamTransferStateDoPrepare(pTask);
} else { } else {
// no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task. // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task.
SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); SStreamTask* pStreamTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask != NULL) { if (pStreamTask != NULL) {
// halt the related stream sink task // halt the related stream sink task
code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL); code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL);
@ -590,12 +599,13 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId); SStreamTask* pHTask = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask != NULL) { if (pHTask != NULL) {
streamTaskReleaseState(pHTask); streamTaskReleaseState(pHTask);
streamTaskReloadState(pTask); streamTaskReloadState(pTask);
stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr,
streamTaskGetStatus(pHTask)->name); streamTaskGetStatus(pHTask).name);
streamMetaReleaseTask(pTask->pMeta, pHTask); streamMetaReleaseTask(pTask->pMeta, pHTask);
} else { } else {
@ -625,7 +635,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
int32_t numOfBlocks = 0; int32_t numOfBlocks = 0;
SStreamQueueItem* pInput = NULL; SStreamQueueItem* pInput = NULL;
if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask)->state == TASK_STATUS__UNINIT)) { if (streamTaskShouldStop(pTask) || (streamTaskGetStatus(pTask).state == TASK_STATUS__UNINIT)) {
stDebug("s-task:%s stream task is stopped", id); stDebug("s-task:%s stream task is stopped", id);
return 0; return 0;
} }
@ -703,9 +713,9 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
} else { // todo other thread may change the status } else { // todo other thread may change the status
// do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed.
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
stDebug("s-task:%s checkpoint block received, set status:%s", id, pState->name); stDebug("s-task:%s checkpoint block received, set status:%s", id, pState.name);
streamTaskBuildCheckpoint(pTask); streamTaskBuildCheckpoint(pTask);
} else { // todo refactor } else { // todo refactor
int32_t code = 0; int32_t code = 0;
@ -732,17 +742,17 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
// the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not // the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not
// be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING. // be updated by tryExec function, therefore, the schedStatus will always be the TASK_SCHED_STATUS__WAITING.
bool streamTaskIsIdle(const SStreamTask* pTask) { bool streamTaskIsIdle(const SStreamTask* pTask) {
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || status == TASK_STATUS__STOP || return (pTask->status.schedStatus == TASK_SCHED_STATUS__INACTIVE || status == TASK_STATUS__STOP ||
status == TASK_STATUS__DROPPING); status == TASK_STATUS__DROPPING);
} }
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) { bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus) {
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
ETaskStatus st = pState->state; ETaskStatus st = pState.state;
if (pStatus != NULL) { if (pStatus != NULL) {
*pStatus = pState->name; *pStatus = pState.name;
} }
// pause & halt will still run for sink tasks. // pause & halt will still run for sink tasks.
@ -773,7 +783,7 @@ int32_t streamResumeTask(SStreamTask* pTask) {
setLastExecTs(pTask, taosGetTimestampMs()); setLastExecTs(pTask, taosGetTimestampMs());
char* p = streamTaskGetStatus(pTask)->name; char* p = streamTaskGetStatus(pTask).name;
stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p, stDebug("s-task:%s exec completed, status:%s, sched-status:%d, lastExecTs:%" PRId64, id, p,
pTask->status.schedStatus, pTask->status.lastExecTs); pTask->status.schedStatus, pTask->status.lastExecTs);
@ -801,7 +811,7 @@ int32_t streamExecTask(SStreamTask* pTask) {
if (schedStatus == TASK_SCHED_STATUS__WAITING) { if (schedStatus == TASK_SCHED_STATUS__WAITING) {
streamResumeTask(pTask); streamResumeTask(pTask);
} else { } else {
char* p = streamTaskGetStatus(pTask)->name; char* p = streamTaskGetStatus(pTask).name;
stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p, stDebug("s-task:%s already started to exec by other thread, status:%s, sched-status:%d", id, p,
pTask->status.schedStatus); pTask->status.schedStatus);
} }

96
source/libs/stream/src/streamHb.c
View File

@ -54,7 +54,7 @@ static bool existInHbMsg(SStreamHbMsg* pMsg, SDownstreamTaskEpset* pTaskEpset) {
static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList); int32_t num = taosArrayGetSize(pTask->outputInfo.pNodeEpsetUpdateList);
for (int j = 0; j < num; ++j) { for (int j = 0; j < num; ++j) {
@ -62,14 +62,18 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) {
bool exist = existInHbMsg(pMsg, pTaskEpset); bool exist = existInHbMsg(pMsg, pTaskEpset);
if (!exist) { if (!exist) {
taosArrayPush(pMsg->pUpdateNodes, &pTaskEpset->nodeId); void* p = taosArrayPush(pMsg->pUpdateNodes, &pTaskEpset->nodeId);
if (p == NULL) {
stError("failed to set the updateNode info in hbMsg, vgId:%d", pMeta->vgId);
}
stDebug("vgId:%d nodeId:%d added into hbMsg update list, total:%d", pMeta->vgId, pTaskEpset->nodeId, stDebug("vgId:%d nodeId:%d added into hbMsg update list, total:%d", pMeta->vgId, pTaskEpset->nodeId,
(int32_t)taosArrayGetSize(pMsg->pUpdateNodes)); (int32_t)taosArrayGetSize(pMsg->pUpdateNodes));
} }
} }
taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList); taosArrayClear(pTask->outputInfo.pNodeEpsetUpdateList);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
} }
static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) { static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* pEpset) {
@ -101,9 +105,7 @@ static int32_t doSendHbMsgInfo(SStreamHbMsg* pMsg, SStreamMeta* pMeta, SEpSet* p
SRpcMsg msg = {0}; SRpcMsg msg = {0};
initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen);
tmsgSendReq(pEpset, &msg); return tmsgSendReq(pEpset, &msg);
return TSDB_CODE_SUCCESS;
} }
// NOTE: this task should be executed within the SStreamMeta lock region. // NOTE: this task should be executed within the SStreamMeta lock region.
@ -112,6 +114,7 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
bool hasMnodeEpset = false; bool hasMnodeEpset = false;
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
SMetaHbInfo* pInfo = pMeta->pHbInfo; SMetaHbInfo* pInfo = pMeta->pHbInfo;
int32_t code = 0;
// not recv the hb msg rsp yet, send current hb msg again // not recv the hb msg rsp yet, send current hb msg again
if (pInfo->msgSendTs > 0) { if (pInfo->msgSendTs > 0) {
@ -135,8 +138,7 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
} }
pInfo->msgSendTs = taosGetTimestampMs(); pInfo->msgSendTs = taosGetTimestampMs();
doSendHbMsgInfo(&pInfo->hbMsg, pMeta, &epset); return doSendHbMsgInfo(&pInfo->hbMsg, pMeta, &epset);
return TSDB_CODE_SUCCESS;
} }
SStreamHbMsg* pMsg = &pInfo->hbMsg; SStreamHbMsg* pMsg = &pInfo->hbMsg;
@ -168,9 +170,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
continue; continue;
} }
taosThreadMutexLock(&(*pTask)->lock); (void) taosThreadMutexLock(&(*pTask)->lock);
STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask); STaskStatusEntry entry = streamTaskGetStatusEntry(*pTask);
taosThreadMutexUnlock(&(*pTask)->lock); (void) taosThreadMutexUnlock(&(*pTask)->lock);
entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE); entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) { if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) {
@ -188,9 +190,9 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo", stInfo("s-task:%s set kill checkpoint trans in hbMsg, transId:%d, clear the active checkpointInfo",
(*pTask)->id.idStr, p->transId); (*pTask)->id.idStr, p->transId);
taosThreadMutexLock(&(*pTask)->lock); (void) taosThreadMutexLock(&(*pTask)->lock);
streamTaskClearCheckInfo((*pTask), true); streamTaskClearCheckInfo((*pTask), true);
taosThreadMutexUnlock(&(*pTask)->lock); (void) taosThreadMutexUnlock(&(*pTask)->lock);
} }
} }
@ -210,7 +212,11 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
} }
addUpdateNodeIntoHbMsg(*pTask, pMsg); addUpdateNodeIntoHbMsg(*pTask, pMsg);
taosArrayPush(pMsg->pTaskStatus, &entry); p = taosArrayPush(pMsg->pTaskStatus, &entry);
if (p == NULL) {
stError("failed to add taskInfo:0x%x in hbMsg, vgId:%d", (*pTask)->id.taskId, pMeta->vgId);
}
if (!hasMnodeEpset) { if (!hasMnodeEpset) {
epsetAssign(&epset, &(*pTask)->info.mnodeEpset); epsetAssign(&epset, &(*pTask)->info.mnodeEpset);
hasMnodeEpset = true; hasMnodeEpset = true;
@ -221,18 +227,19 @@ int32_t streamMetaSendHbHelper(SStreamMeta* pMeta) {
if (hasMnodeEpset) { if (hasMnodeEpset) {
pInfo->msgSendTs = taosGetTimestampMs(); pInfo->msgSendTs = taosGetTimestampMs();
doSendHbMsgInfo(pMsg, pMeta, &epset); code = doSendHbMsgInfo(pMsg, pMeta, &epset);
} else { } else {
stDebug("vgId:%d no tasks or no mnd epset, not send stream hb to mnode", pMeta->vgId); stDebug("vgId:%d no tasks or no mnd epset, not send stream hb to mnode", pMeta->vgId);
tCleanupStreamHbMsg(&pInfo->hbMsg); tCleanupStreamHbMsg(&pInfo->hbMsg);
pInfo->msgSendTs = -1; pInfo->msgSendTs = -1;
} }
return TSDB_CODE_SUCCESS; return code;
} }
void streamMetaHbToMnode(void* param, void* tmrId) { void streamMetaHbToMnode(void* param, void* tmrId) {
int64_t rid = *(int64_t*)param; int64_t rid = *(int64_t*)param;
int32_t code = 0;
SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid);
if (pMeta == NULL) { if (pMeta == NULL) {
@ -243,15 +250,25 @@ void streamMetaHbToMnode(void* param, void* tmrId) {
// need to stop, stop now // need to stop, stop now
if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { // todo refactor: not need this now, use closeFlag in Meta if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { // todo refactor: not need this now, use closeFlag in Meta
pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP; pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP;
stDebug("vgId:%d jump out of meta timer", pMeta->vgId); code = taosReleaseRef(streamMetaId, rid);
taosReleaseRef(streamMetaId, rid); if (code == TSDB_CODE_SUCCESS) {
stDebug("vgId:%d jump out of meta timer", pMeta->vgId);
} else {
stError("vgId:%d jump out of meta timer, failed to release the meta rid:%" PRId64, pMeta->vgId, rid);
}
return; return;
} }
// not leader not send msg // not leader not send msg
if (pMeta->role != NODE_ROLE_LEADER) { if (pMeta->role != NODE_ROLE_LEADER) {
stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); code = taosReleaseRef(streamMetaId, rid);
taosReleaseRef(streamMetaId, rid); if (code == TSDB_CODE_SUCCESS) {
stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role);
} else {
stError("vgId:%d role:%d not leader not send hb to mnodefailed to release the meta rid:%" PRId64, pMeta->vgId,
pMeta->role, rid);
}
pMeta->pHbInfo->hbStart = 0; pMeta->pHbInfo->hbStart = 0;
return; return;
} }
@ -262,24 +279,37 @@ void streamMetaHbToMnode(void* param, void* tmrId) {
} }
if (!waitForEnoughDuration(pMeta->pHbInfo)) { if (!waitForEnoughDuration(pMeta->pHbInfo)) {
taosTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId,
taosReleaseRef(streamMetaId, rid); "meta-hb-tmr");
code = taosReleaseRef(streamMetaId, rid);
if (code) {
stError("vgId:%d in meta timer, failed to release the meta rid:%" PRId64, pMeta->vgId, rid);
}
return; return;
} }
streamMetaRLock(pMeta); streamMetaRLock(pMeta);
streamMetaSendHbHelper(pMeta); code = streamMetaSendHbHelper(pMeta);
streamMetaRUnLock(pMeta); if (code) {
stError("vgId:%d failed to send hmMsg to mnode, try again in 5s, code:%s", pMeta->vgId, strerror(code));
}
taosTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); streamMetaRUnLock(pMeta);
taosReleaseRef(streamMetaId, rid); streamTmrReset(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr, pMeta->vgId,
"meta-hb-tmr");
code = taosReleaseRef(streamMetaId, rid);
if (code) {
stError("vgId:%d in meta timer, failed to release the meta rid:%" PRId64, pMeta->vgId, rid);
}
} }
SMetaHbInfo* createMetaHbInfo(int64_t* pRid) { int32_t createMetaHbInfo(int64_t* pRid, SMetaHbInfo** pRes) {
*pRes = NULL;
SMetaHbInfo* pInfo = taosMemoryCalloc(1, sizeof(SMetaHbInfo)); SMetaHbInfo* pInfo = taosMemoryCalloc(1, sizeof(SMetaHbInfo));
if (pInfo == NULL) { if (pInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return pInfo;
} }
pInfo->hbTmr = taosTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer); pInfo->hbTmr = taosTmrStart(streamMetaHbToMnode, META_HB_CHECK_INTERVAL, pRid, streamTimer);
@ -287,22 +317,22 @@ SMetaHbInfo* createMetaHbInfo(int64_t* pRid) {
pInfo->stopFlag = 0; pInfo->stopFlag = 0;
pInfo->msgSendTs = -1; pInfo->msgSendTs = -1;
pInfo->hbCount = 0; pInfo->hbCount = 0;
return pInfo;
*pRes = pInfo;
return TSDB_CODE_SUCCESS;
} }
void* destroyMetaHbInfo(SMetaHbInfo* pInfo) { void destroyMetaHbInfo(SMetaHbInfo* pInfo) {
if (pInfo != NULL) { if (pInfo != NULL) {
tCleanupStreamHbMsg(&pInfo->hbMsg); tCleanupStreamHbMsg(&pInfo->hbMsg);
if (pInfo->hbTmr != NULL) { if (pInfo->hbTmr != NULL) {
taosTmrStop(pInfo->hbTmr); (void) taosTmrStop(pInfo->hbTmr);
pInfo->hbTmr = NULL; pInfo->hbTmr = NULL;
} }
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} }
return NULL;
} }
void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta) { void streamMetaWaitForHbTmrQuit(SStreamMeta* pMeta) {

138
source/libs/stream/src/streamMeta.c
View File

@ -292,32 +292,39 @@ void streamMetaRemoveDB(void* arg, char* key) {
taosThreadMutexUnlock(&pMeta->backendMutex); taosThreadMutexUnlock(&pMeta->backendMutex);
} }
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, int32_t vgId,
int32_t vgId, int64_t stage, startComplete_fn_t fn) { int64_t stage, startComplete_fn_t fn, SStreamMeta** p) {
*p = NULL;
int32_t code = 0;
SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
if (pMeta == NULL) { if (pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("vgId:%d failed to prepare stream meta, alloc size:%" PRIzu ", out of memory", vgId, sizeof(SStreamMeta)); stError("vgId:%d failed to prepare stream meta, alloc size:%" PRIzu ", out of memory", vgId, sizeof(SStreamMeta));
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t len = strlen(path) + 64; int32_t len = strlen(path) + 64;
char* tpath = taosMemoryCalloc(1, len); char* tpath = taosMemoryCalloc(1, len);
if (tpath == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream"); sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream");
pMeta->path = tpath; pMeta->path = tpath;
if (streamMetaOpenTdb(pMeta) < 0) { code = streamMetaOpenTdb(pMeta);
if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _err;
} }
if (streamMetaMayCvtDbFormat(pMeta) < 0) { if ((code = streamMetaMayCvtDbFormat(pMeta)) < 0) {
stError("vgId:%d convert sub info format failed, open stream meta failed, reason: %s", pMeta->vgId, stError("vgId:%d convert sub info format failed, open stream meta failed, reason: %s", pMeta->vgId,
tstrerror(terrno)); tstrerror(terrno));
goto _err; goto _err;
} }
if (streamMetaBegin(pMeta) < 0) { if ((code = streamMetaBegin(pMeta) < 0)) {
stError("vgId:%d begin trans for stream meta failed", pMeta->vgId); stError("vgId:%d begin trans for stream meta failed", pMeta->vgId);
goto _err; goto _err;
} }
@ -325,32 +332,32 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas
_hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
pMeta->pTasksMap = taosHashInit(64, fp, true, HASH_NO_LOCK); pMeta->pTasksMap = taosHashInit(64, fp, true, HASH_NO_LOCK);
if (pMeta->pTasksMap == NULL) { if (pMeta->pTasksMap == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK); pMeta->updateInfo.pTasks = taosHashInit(64, fp, false, HASH_NO_LOCK);
if (pMeta->updateInfo.pTasks == NULL) { if (pMeta->updateInfo.pTasks == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
pMeta->startInfo.pReadyTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK); pMeta->startInfo.pReadyTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK);
if (pMeta->startInfo.pReadyTaskSet == NULL) { if (pMeta->startInfo.pReadyTaskSet == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
pMeta->startInfo.pFailedTaskSet = taosHashInit(4, fp, false, HASH_NO_LOCK); pMeta->startInfo.pFailedTaskSet = taosHashInit(4, fp, false, HASH_NO_LOCK);
if (pMeta->startInfo.pFailedTaskSet == NULL) { if (pMeta->startInfo.pFailedTaskSet == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
// task list // task list
pMeta->pTaskList = taosArrayInit(4, sizeof(SStreamTaskId)); pMeta->pTaskList = taosArrayInit(4, sizeof(SStreamTaskId));
if (pMeta->pTaskList == NULL) { if (pMeta->pTaskList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -376,10 +383,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas
// set the attribute when running on Linux OS // set the attribute when running on Linux OS
TdThreadRwlockAttr attr; TdThreadRwlockAttr attr;
taosThreadRwlockAttrInit(&attr); code = taosThreadRwlockAttrInit(&attr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
#ifdef LINUX #ifdef LINUX
pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); code = pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
#endif #endif
taosThreadRwlockInit(&pMeta->lock, &attr); taosThreadRwlockInit(&pMeta->lock, &attr);
@ -389,9 +402,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas
memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid));
metaRefMgtAdd(pMeta->vgId, pRid); metaRefMgtAdd(pMeta->vgId, pRid);
pMeta->pHbInfo = createMetaHbInfo(pRid); code = createMetaHbInfo(pRid, &pMeta->pHbInfo);
if (pMeta->pHbInfo == NULL) { if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
} }
@ -403,7 +415,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTas
} }
taosThreadMutexInit(&pMeta->backendMutex, NULL); taosThreadMutexInit(&pMeta->backendMutex, NULL);
return pMeta; *p = pMeta;
return code;
_err: _err:
taosMemoryFree(pMeta->path); taosMemoryFree(pMeta->path);
@ -420,7 +433,7 @@ _err:
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
stError("failed to open stream meta, reason:%s", tstrerror(terrno)); stError("failed to open stream meta, reason:%s", tstrerror(terrno));
return NULL; return code;
} }
// todo refactor: the lock shoud be restricted in one function // todo refactor: the lock shoud be restricted in one function
@ -524,7 +537,8 @@ void streamMetaCloseImpl(void* arg) {
taosHashCleanup(pMeta->startInfo.pReadyTaskSet); taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
taosHashCleanup(pMeta->startInfo.pFailedTaskSet); taosHashCleanup(pMeta->startInfo.pFailedTaskSet);
pMeta->pHbInfo = destroyMetaHbInfo(pMeta->pHbInfo); destroyMetaHbInfo(pMeta->pHbInfo);
pMeta->pHbInfo = NULL;
taosMemoryFree(pMeta->path); taosMemoryFree(pMeta->path);
taosThreadMutexDestroy(&pMeta->backendMutex); taosThreadMutexDestroy(&pMeta->backendMutex);
@ -630,29 +644,30 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) {
return (int32_t)size; return (int32_t)size;
} }
SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask) {
STaskId id = {.streamId = streamId, .taskId = taskId}; STaskId id = {.streamId = streamId, .taskId = taskId};
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL || streamTaskShouldStop(*ppTask)) { if (ppTask == NULL || streamTaskShouldStop(*ppTask)) {
return NULL; *pTask = NULL;
return TSDB_CODE_FAILED;
} }
int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1);
stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref);
return *ppTask; *pTask = *ppTask;
return TSDB_CODE_SUCCESS;
} }
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask) {
streamMetaRLock(pMeta); streamMetaRLock(pMeta);
SStreamTask* p = streamMetaAcquireTaskNoLock(pMeta, streamId, taskId); int32_t code = streamMetaAcquireTaskNoLock(pMeta, streamId, taskId, pTask);
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
return p; return code;
} }
SStreamTask* streamMetaAcquireOneTask(SStreamTask* pTask) { void streamMetaAcquireOneTask(SStreamTask* pTask) {
int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1);
stTrace("s-task:%s acquire task, ref:%d", pTask->id.idStr, ref); stTrace("s-task:%s acquire task, ref:%d", pTask->id.idStr, ref);
return pTask;
} }
void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) {
@ -719,20 +734,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, pMeta->vgId); stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, pMeta->vgId);
while (1) { while (1) {
streamMetaRLock(pMeta); int32_t timerActive = 0;
streamMetaRLock(pMeta);
ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask) { if (ppTask) {
if ((*ppTask)->status.timerActive == 0) { timerActive = (*ppTask)->status.timerActive;
streamMetaRUnLock(pMeta); }
break; streamMetaRUnLock(pMeta);
}
taosMsleep(10); if (timerActive > 0) {
stDebug("s-task:%s wait for quit from timer", (*ppTask)->id.idStr); taosMsleep(100);
streamMetaRUnLock(pMeta); stDebug("s-task:0x%" PRIx64 " wait for quit from timer", id.taskId);
} else { } else {
streamMetaRUnLock(pMeta);
break; break;
} }
} }
@ -922,7 +936,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
continue; continue;
} }
if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) < 0) { if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) {
stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno)); stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno));
taosArrayPop(pMeta->pTaskList); taosArrayPop(pMeta->pTaskList);
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
@ -1090,13 +1104,19 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi
return taosScheduleTask(pMeta->qHandle, &schedMsg); return taosScheduleTask(pMeta->qHandle, &schedMsg);
} }
SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) {
*pList = NULL;
int32_t code = 0;
SArray* pTaskList = taosArrayDup(pMeta->pTaskList, NULL); SArray* pTaskList = taosArrayDup(pMeta->pTaskList, NULL);
if (pTaskList == NULL) {
stError("failed to generate the task list during send hbMsg to mnode, vgId:%d, code: out of memory", pMeta->vgId);
return TSDB_CODE_OUT_OF_MEMORY;
}
bool sendMsg = pMeta->sendMsgBeforeClosing; bool sendMsg = pMeta->sendMsgBeforeClosing;
if (!sendMsg) { if (!sendMsg) {
stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId); stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId);
return pTaskList; return TSDB_CODE_SUCCESS;
} }
stDebug("vgId:%d send msg to mnode before closing all tasks", pMeta->vgId); stDebug("vgId:%d send msg to mnode before closing all tasks", pMeta->vgId);
@ -1105,27 +1125,29 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) {
int32_t numOfTasks = taosArrayGetSize(pTaskList); int32_t numOfTasks = taosArrayGetSize(pTaskList);
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); SStreamTask* pTask = NULL;
if (pTask == NULL) {
code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (code != TSDB_CODE_SUCCESS) { // this error is ignored
continue; continue;
} }
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state == TASK_STATUS__CK) { if (pState.state == TASK_STATUS__CK) {
streamTaskSetFailedCheckpointId(pTask); streamTaskSetFailedCheckpointId(pTask);
} else { } else {
stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState->name); stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} }
streamMetaSendHbHelper(pMeta); code = streamMetaSendHbHelper(pMeta);
pMeta->sendMsgBeforeClosing = false; pMeta->sendMsgBeforeClosing = false;
return pTaskList; return code;
} }
void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) { void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) {
@ -1201,7 +1223,8 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
// initialization, when the operation of check downstream tasks status is executed far quickly. // initialization, when the operation of check downstream tasks status is executed far quickly.
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId);
@ -1223,7 +1246,8 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId);
@ -1293,13 +1317,20 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) {
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
// send hb msg to mnode before closing all tasks. // send hb msg to mnode before closing all tasks.
SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); SArray* pTaskList = NULL;
int32_t code = streamMetaSendMsgBeforeCloseTasks(pMeta, &pTaskList);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
int32_t numOfTasks = taosArrayGetSize(pTaskList); int32_t numOfTasks = taosArrayGetSize(pTaskList);
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); SStreamTask* pTask = NULL;
if (pTask == NULL) {
code = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (code != TSDB_CODE_SUCCESS) {
continue; continue;
} }
@ -1339,7 +1370,8 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId); stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId); stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId);
streamMetaAddFailedTask(pMeta, streamId, taskId); streamMetaAddFailedTask(pMeta, streamId, taskId);

48
source/libs/stream/src/streamMsg.c
View File

@ -99,6 +99,8 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda
} }
int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) { int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) {
int32_t code = 0;
if (tStartDecode(pDecoder) < 0) return -1; if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pMsg->streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pMsg->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pMsg->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pMsg->taskId) < 0) return -1;
@ -111,13 +113,17 @@ int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg*
if (tDecodeI32(pDecoder, &info.nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &info.nodeId) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &info.prevEp) < 0) return -1; if (tDecodeSEpSet(pDecoder, &info.prevEp) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &info.newEp) < 0) return -1; if (tDecodeSEpSet(pDecoder, &info.newEp) < 0) return -1;
taosArrayPush(pMsg->pNodeList, &info);
void* p = taosArrayPush(pMsg->pNodeList, &info);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (tDecodeI32(pDecoder, &pMsg->transId) < 0) return -1; if (tDecodeI32(pDecoder, &pMsg->transId) < 0) return -1;
tEndDecode(pDecoder); tEndDecode(pDecoder);
return 0; return code;
} }
int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) { int32_t tEncodeStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) {
@ -257,8 +263,18 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
if (tDecodeI32(pDecoder, &len1) < 0) return -1; if (tDecodeI32(pDecoder, &len1) < 0) return -1;
if (tDecodeBinaryAlloc(pDecoder, &data, &len2) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &data, &len2) < 0) return -1;
ASSERT(len1 == len2); ASSERT(len1 == len2);
taosArrayPush(pReq->dataLen, &len1);
taosArrayPush(pReq->data, &data); void* p = taosArrayPush(pReq->dataLen, &len1);
if (p == NULL) {
tEndDecode(pDecoder);
return TSDB_CODE_OUT_OF_MEMORY;
}
p = taosArrayPush(pReq->data, &data);
if (p == NULL) {
tEndDecode(pDecoder);
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
tEndDecode(pDecoder); tEndDecode(pDecoder);
@ -371,6 +387,8 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) {
} }
int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
int32_t code = 0;
if (tStartDecode(pDecoder) < 0) return -1; if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->vgId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->numOfTasks) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->numOfTasks) < 0) return -1;
@ -413,7 +431,11 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
if (tDecodeI64(pDecoder, &entry.hTaskId) < 0) return -1; if (tDecodeI64(pDecoder, &entry.hTaskId) < 0) return -1;
entry.id.taskId = taskId; entry.id.taskId = taskId;
taosArrayPush(pReq->pTaskStatus, &entry); void* p = taosArrayPush(pReq->pTaskStatus, &entry);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
} }
int32_t numOfVgs = 0; int32_t numOfVgs = 0;
@ -424,12 +446,20 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) {
for (int j = 0; j < numOfVgs; ++j) { for (int j = 0; j < numOfVgs; ++j) {
int32_t vgId = 0; int32_t vgId = 0;
if (tDecodeI32(pDecoder, &vgId) < 0) return -1; if (tDecodeI32(pDecoder, &vgId) < 0) return -1;
taosArrayPush(pReq->pUpdateNodes, &vgId); void* p = taosArrayPush(pReq->pUpdateNodes, &vgId);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
} }
if (tDecodeI32(pDecoder, &pReq->msgId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->msgId) < 0) return -1;
tEndDecode(pDecoder); tEndDecode(pDecoder);
return 0; return 0;
_err:
tEndDecode(pDecoder);
return code;
} }
void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) { void tCleanupStreamHbMsg(SStreamHbMsg* pMsg) {
@ -572,7 +602,11 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pInfo);
return -1; return -1;
} }
taosArrayPush(pTask->upstreamInfo.pList, &pInfo); void* p = taosArrayPush(pTask->upstreamInfo.pList, &pInfo);
if (p == NULL) {
tEndDecode(pDecoder);
return -1;
}
} }
if (pTask->info.taskLevel != TASK_LEVEL__SINK) { if (pTask->info.taskLevel != TASK_LEVEL__SINK) {

29
source/libs/stream/src/streamQueue.c
View File

@ -41,34 +41,34 @@ static void streamQueueCleanup(SStreamQueue* pQueue) {
static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } static void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; }
SStreamQueue* streamQueueOpen(int64_t cap) { int32_t streamQueueOpen(int64_t cap, SStreamQueue** pQ) {
int32_t code; *pQ = NULL;
int32_t code = 0;
SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue)); SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue));
if (pQueue == NULL) { if (pQueue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
code = taosOpenQueue(&pQueue->pQueue); code = taosOpenQueue(&pQueue->pQueue);
if (code) { if (code) {
taosMemoryFreeClear(pQueue); taosMemoryFreeClear(pQueue);
terrno = code; return code;
return NULL;
} }
code = taosAllocateQall(&pQueue->qall); code = taosAllocateQall(&pQueue->qall);
if (code) { if (code) {
taosCloseQueue(pQueue->pQueue); taosCloseQueue(pQueue->pQueue);
taosMemoryFree(pQueue); taosMemoryFree(pQueue);
terrno = code; return code;
return NULL;
} }
pQueue->status = STREAM_QUEUE__SUCESS; pQueue->status = STREAM_QUEUE__SUCESS;
taosSetQueueCapacity(pQueue->pQueue, cap); taosSetQueueCapacity(pQueue->pQueue, cap);
taosSetQueueMemoryCapacity(pQueue->pQueue, cap * 1024); taosSetQueueMemoryCapacity(pQueue->pQueue, cap * 1024);
return pQueue;
*pQ = pQueue;
return code;
} }
void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) { void streamQueueClose(SStreamQueue* pQueue, int32_t taskId) {
@ -227,12 +227,11 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte
*pInput = qItem; *pInput = qItem;
} else { } else {
// merge current block failed, let's handle the already merged blocks. // merge current block failed, let's handle the already merged blocks.
void* newRet = streamQueueMergeQueueItem(*pInput, qItem); void* newRet = NULL;
if (newRet == NULL) { int32_t code = streamQueueMergeQueueItem(*pInput, qItem, (SStreamQueueItem**)&newRet);
if (terrno != 0) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks,
tstrerror(terrno)); tstrerror(terrno));
}
*blockSize = streamQueueItemGetSize(*pInput); *blockSize = streamQueueItemGetSize(*pInput);
if (taskLevel == TASK_LEVEL__SINK) { if (taskLevel == TASK_LEVEL__SINK) {

8
source/libs/stream/src/streamSched.c
View File

@ -94,13 +94,13 @@ int32_t streamTaskResumeInFuture(SStreamTask* pTask) {
void streamTaskResumeHelper(void* param, void* tmrId) { void streamTaskResumeHelper(void* param, void* tmrId) {
SStreamTask* pTask = (SStreamTask*)param; SStreamTask* pTask = (SStreamTask*)param;
SStreamTaskId* pId = &pTask->id; SStreamTaskId* pId = &pTask->id;
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) { if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) {
streamTaskSetSchedStatusInactive(pTask); streamTaskSetSchedStatusInactive(pTask);
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p->name, ref); stDebug("s-task:%s status:%s not resume task, ref:%d", pId->idStr, p.name, ref);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;
@ -130,7 +130,7 @@ void streamTaskSchedHelper(void* param, void* tmrId) {
return; return;
} }
if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { if (streamTaskGetStatus(pTask).state == TASK_STATUS__CK) {
stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger); stDebug("s-task:%s in checkpoint procedure, not retrieve result, next:%dms", id, nextTrigger);
} else { } else {
if (status == TASK_TRIGGER_STATUS__ACTIVE) { if (status == TASK_TRIGGER_STATUS__ACTIVE) {

61
source/libs/stream/src/streamStartHistory.c
View File

@ -44,13 +44,13 @@ static void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunc
static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now);
static int32_t streamTaskSetReady(SStreamTask* pTask) { static int32_t streamTaskSetReady(SStreamTask* pTask) {
int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask);
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if ((p->state == TASK_STATUS__SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { if ((p.state == TASK_STATUS__SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
int32_t numOfUps = taosArrayGetSize(pTask->upstreamInfo.pList); int32_t numOfUps = taosArrayGetSize(pTask->upstreamInfo.pList);
stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s", stDebug("s-task:%s level:%d task wait for %d upstream tasks complete scan-history procedure, status:%s",
pTask->id.idStr, pTask->info.taskLevel, numOfUps, p->name); pTask->id.idStr, pTask->info.taskLevel, numOfUps, p.name);
} }
ASSERT(pTask->status.downstreamReady == 0); ASSERT(pTask->status.downstreamReady == 0);
@ -59,7 +59,7 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) {
pTask->execInfo.readyTs = taosGetTimestampMs(); pTask->execInfo.readyTs = taosGetTimestampMs();
int64_t el = (pTask->execInfo.readyTs - pTask->execInfo.checkTs); int64_t el = (pTask->execInfo.readyTs - pTask->execInfo.checkTs);
stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s",
pTask->id.idStr, numOfDowns, el, p->name); pTask->id.idStr, numOfDowns, el, p.name);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -92,10 +92,11 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration)
} }
// add ref for task // add ref for task
SStreamTask* p = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId); SStreamTask* p = NULL;
int32_t code = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, &p);
if (p == NULL) { if (p == NULL) {
stError("s-task:0x%x failed to acquire task, status:%s, not exec scan-history data", pTask->id.taskId, stError("s-task:0x%x failed to acquire task, status:%s, not exec scan-history data", pTask->id.taskId,
streamTaskGetStatus(pTask)->name); streamTaskGetStatus(pTask).name);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -117,7 +118,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration)
int32_t streamTaskStartScanHistory(SStreamTask* pTask) { int32_t streamTaskStartScanHistory(SStreamTask* pTask) {
int32_t level = pTask->info.taskLevel; int32_t level = pTask->info.taskLevel;
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
ASSERT((pTask->status.downstreamReady == 1) && (status == TASK_STATUS__SCAN_HISTORY) && ASSERT((pTask->status.downstreamReady == 1) && (status == TASK_STATUS__SCAN_HISTORY) &&
(pTask->info.fillHistory == 1)); (pTask->info.fillHistory == 1));
@ -138,8 +139,8 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
streamTaskSetReady(pTask); streamTaskSetReady(pTask);
streamTaskSetRangeStreamCalc(pTask); streamTaskSetRangeStreamCalc(pTask);
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
ASSERT(p->state == TASK_STATUS__READY); ASSERT(p.state == TASK_STATUS__READY);
int8_t schedStatus = pTask->status.schedStatus; int8_t schedStatus = pTask->status.schedStatus;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -148,10 +149,10 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
startVer = pTask->chkInfo.nextProcessVer; startVer = pTask->chkInfo.nextProcessVer;
} }
stDebug("s-task:%s status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, id, p->name, schedStatus, stDebug("s-task:%s status:%s, sched-status:%d, ready for data from wal ver:%" PRId64, id, p.name, schedStatus,
startVer); startVer);
} else { } else {
stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p->name, schedStatus); stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p.name, schedStatus);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -162,10 +163,10 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
streamTaskSetReady(pTask); streamTaskSetReady(pTask);
streamTaskSetRangeStreamCalc(pTask); streamTaskSetRangeStreamCalc(pTask);
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
ASSERT((p->state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1));
stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p->name); stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name);
streamTaskStartScanHistory(pTask); streamTaskStartScanHistory(pTask);
// NOTE: there will be an deadlock if launch fill history here. // NOTE: there will be an deadlock if launch fill history here.
@ -205,11 +206,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
ASSERT(hTaskId != 0); ASSERT(hTaskId != 0);
// check stream task status in the first place. // check stream task status in the first place.
SStreamTaskState* pStatus = streamTaskGetStatus(pTask); SStreamTaskState pStatus = streamTaskGetStatus(pTask);
if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT && if (pStatus.state != TASK_STATUS__READY && pStatus.state != TASK_STATUS__HALT &&
pStatus->state != TASK_STATUS__PAUSE) { pStatus.state != TASK_STATUS__PAUSE) {
stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId,
pStatus->name); pStatus.name);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return -1; // todo set the correct error code return -1; // todo set the correct error code
@ -223,7 +224,8 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
streamMetaRUnLock(pMeta); streamMetaRUnLock(pMeta);
if (pHTask != NULL) { // it is already added into stream meta store. if (pHTask != NULL) { // it is already added into stream meta store.
SStreamTask* pHisTask = streamMetaAcquireTask(pMeta, hStreamId, hTaskId); SStreamTask* pHisTask = NULL;
code = streamMetaAcquireTask(pMeta, hStreamId, hTaskId, &pHisTask);
if (pHisTask == NULL) { if (pHisTask == NULL) {
stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr); stDebug("s-task:%s failed acquire and start fill-history task, it may have been dropped/stopped", idStr);
streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
@ -308,7 +310,7 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i
streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false); streamMetaAddTaskLaunchResult(pMeta, pInfo->hTaskId.streamId, pInfo->hTaskId.taskId, 0, now, false);
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} else { } else {
char* p = streamTaskGetStatus(pTask)->name; char* p = streamTaskGetStatus(pTask).name;
int32_t hTaskId = pHTaskInfo->id.taskId; int32_t hTaskId = pHTaskInfo->id.taskId;
stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d",
@ -340,7 +342,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
if (streamTaskShouldStop(*ppTask)) { if (streamTaskShouldStop(*ppTask)) {
ASSERT((*ppTask)->status.timerActive >= 1); ASSERT((*ppTask)->status.timerActive >= 1);
char* p = streamTaskGetStatus(*ppTask)->name; char* p = streamTaskGetStatus(*ppTask).name;
int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&(*ppTask)->status.timerActive, 1);
stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d", stDebug("s-task:%s status:%s should stop, quit launch fill-history task timer, retry:%d, ref:%d",
(*ppTask)->id.idStr, p, (*ppTask)->hTaskInfo.retryTimes, ref); (*ppTask)->id.idStr, p, (*ppTask)->hTaskInfo.retryTimes, ref);
@ -353,7 +355,11 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
return; return;
} }
SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pInfo->id.streamId, pInfo->id.taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTaskNoLock(pMeta, pInfo->id.streamId, pInfo->id.taskId, &pTask);
if (code != TSDB_CODE_SUCCESS) {
// todo
}
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
if (pTask != NULL) { if (pTask != NULL) {
@ -373,7 +379,8 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
ASSERT(pTask->status.timerActive >= 1); ASSERT(pTask->status.timerActive >= 1);
// abort the timer if intend to stop task // abort the timer if intend to stop task
SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId); SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pMeta, pHTaskInfo->id.streamId, pHTaskInfo->id.taskId, &pHTask);
if (pHTask == NULL) { if (pHTask == NULL) {
doRetryLaunchFillHistoryTask(pTask, pInfo, now); doRetryLaunchFillHistoryTask(pTask, pInfo, now);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -537,10 +544,10 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) {
SStreamTask* pTask = param; SStreamTask* pTask = param;
pTask->schedHistoryInfo.numOfTicks -= 1; pTask->schedHistoryInfo.numOfTicks -= 1;
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
if (p->state == TASK_STATUS__DROPPING || p->state == TASK_STATUS__STOP) { if (p.state == TASK_STATUS__DROPPING || p.state == TASK_STATUS__STOP) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p->name, ref); stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p.name, ref);
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
return; return;

116
source/libs/stream/src/streamTask.c
View File

@ -91,14 +91,15 @@ static SStreamUpstreamEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) {
return pEpInfo; return pEpInfo;
} }
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5) { SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** p) {
*p = NULL;
SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) { if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("s-task:0x%" PRIx64 " failed malloc new stream task, size:%d, code:%s", streamId, stError("s-task:0x%" PRIx64 " failed malloc new stream task, size:%d, code:%s", streamId,
(int32_t)sizeof(SStreamTask), tstrerror(terrno)); (int32_t)sizeof(SStreamTask), tstrerror(terrno));
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pTask->ver = SSTREAM_TASK_VER; pTask->ver = SSTREAM_TASK_VER;
@ -110,10 +111,10 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
pTask->info.delaySchedParam = triggerParam; pTask->info.delaySchedParam = triggerParam;
pTask->subtableWithoutMd5 = subtableWithoutMd5; pTask->subtableWithoutMd5 = subtableWithoutMd5;
pTask->status.pSM = streamCreateStateMachine(pTask); int32_t code = streamCreateStateMachine(pTask);
if (pTask->status.pSM == NULL) { if (pTask->status.pSM == NULL || code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pTask); taosMemoryFreeClear(pTask);
return NULL; return code;
} }
char buf[128] = {0}; char buf[128] = {0};
@ -135,7 +136,9 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
epsetAssign(&(pTask->info.mnodeEpset), pEpset); epsetAssign(&(pTask->info.mnodeEpset), pEpset);
addToTaskset(pTaskList, pTask); addToTaskset(pTaskList, pTask);
return pTask; *p = pTask;
return code;
} }
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) { int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo) {
@ -196,9 +199,9 @@ void tFreeStreamTask(SStreamTask* pTask) {
ETaskStatus status1 = TASK_STATUS__UNINIT; ETaskStatus status1 = TASK_STATUS__UNINIT;
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
if (pTask->status.pSM != NULL) { if (pTask->status.pSM != NULL) {
SStreamTaskState* pStatus = streamTaskGetStatus(pTask); SStreamTaskState pStatus = streamTaskGetStatus(pTask);
p = pStatus->name; p = pStatus.name;
status1 = pStatus->state; status1 = pStatus.state;
} }
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
@ -275,7 +278,9 @@ void tFreeStreamTask(SStreamTask* pTask) {
tSimpleHashCleanup(pTask->pNameMap); tSimpleHashCleanup(pTask->pNameMap);
} }
pTask->status.pSM = streamDestroyStateMachine(pTask->status.pSM); streamDestroyStateMachine(pTask->status.pSM);
pTask->status.pSM = NULL;
streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo);
taosMemoryFree(pTask->outputInfo.pTokenBucket); taosMemoryFree(pTask->outputInfo.pTokenBucket);
@ -355,26 +360,27 @@ static void setInitialVersionInfo(SStreamTask* pTask, int64_t ver) {
} }
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver) { int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver) {
pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); (void) createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId, &pTask->id.idStr);
pTask->refCnt = 1; pTask->refCnt = 1;
pTask->inputq.status = TASK_INPUT_STATUS__NORMAL; pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
pTask->inputq.queue = streamQueueOpen(512 << 10); int32_t code1 = streamQueueOpen(512 << 10, &pTask->inputq.queue);
pTask->outputq.queue = streamQueueOpen(512 << 10); int32_t code2 = streamQueueOpen(512 << 10, &pTask->outputq.queue);
if (pTask->inputq.queue == NULL || pTask->outputq.queue == NULL) { if (code1 || code2) {
stError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr); stError("s-task:%s failed to prepare the input/output queue, initialize task failed", pTask->id.idStr);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
pTask->status.timerActive = 0; pTask->status.timerActive = 0;
pTask->status.pSM = streamCreateStateMachine(pTask);
if (pTask->status.pSM == NULL) { int32_t code = streamCreateStateMachine(pTask);
if (pTask->status.pSM == NULL || code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr, stError("s-task:%s failed create state-machine for stream task, initialization failed, code:%s", pTask->id.idStr,
tstrerror(terrno)); tstrerror(code));
return terrno; return code;
} }
pTask->execInfo.created = taosGetTimestampMs(); pTask->execInfo.created = taosGetTimestampMs();
@ -392,7 +398,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
TdThreadMutexAttr attr = {0}; TdThreadMutexAttr attr = {0};
int code = taosThreadMutexAttrInit(&attr); code = taosThreadMutexAttrInit(&attr);
if (code != 0) { if (code != 0) {
stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code)); stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code));
return code; return code;
@ -434,10 +440,10 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
} }
if (pTask->chkInfo.pActiveInfo == NULL) { if (pTask->chkInfo.pActiveInfo == NULL) {
pTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) { int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) {
@ -637,7 +643,13 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) {
} }
void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); SStreamUpstreamEpInfo* pInfo = NULL;
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return;
}
if ((pInfo != NULL) && pInfo->dataAllowed) { if ((pInfo != NULL) && pInfo->dataAllowed) {
pInfo->dataAllowed = false; pInfo->dataAllowed = false;
int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
@ -646,8 +658,14 @@ void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) {
} }
void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) {
SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); SStreamUpstreamEpInfo* pInfo = NULL;
if ((pInfo != NULL) && (!pInfo->dataAllowed)) {
int32_t code = streamTaskGetUpstreamTaskEpInfo(pTask, taskId, &pInfo);
if (code != TSDB_CODE_SUCCESS) {
return;
}
if (pInfo != NULL && (!pInfo->dataAllowed)) {
int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1);
ASSERT(t >= 0); ASSERT(t >= 0);
pInfo->dataAllowed = true; pInfo->dataAllowed = true;
@ -711,7 +729,7 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
if (resetRelHalt) { if (resetRelHalt) {
stDebug("s-task:0x%" PRIx64 " set the persistent status attr to be ready, prev:%s, status in sm:%s", stDebug("s-task:0x%" PRIx64 " set the persistent status attr to be ready, prev:%s, status in sm:%s",
sTaskId.taskId, streamTaskGetStatusStr((*ppStreamTask)->status.taskStatus), sTaskId.taskId, streamTaskGetStatusStr((*ppStreamTask)->status.taskStatus),
streamTaskGetStatus(*ppStreamTask)->name); streamTaskGetStatus(*ppStreamTask).name);
(*ppStreamTask)->status.taskStatus = TASK_STATUS__READY; (*ppStreamTask)->status.taskStatus = TASK_STATUS__READY;
} }
@ -844,7 +862,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
STaskStatusEntry entry = { STaskStatusEntry entry = {
.id = streamTaskGetTaskId(pTask), .id = streamTaskGetTaskId(pTask),
.status = streamTaskGetStatus(pTask)->state, .status = streamTaskGetStatus(pTask).state,
.nodeId = pMeta->vgId, .nodeId = pMeta->vgId,
.stage = pMeta->stage, .stage = pMeta->stage,
@ -889,12 +907,12 @@ void streamTaskPause(SStreamTask* pTask) {
} }
void streamTaskResume(SStreamTask* pTask) { void streamTaskResume(SStreamTask* pTask) {
SStreamTaskState prevState = *streamTaskGetStatus(pTask); SStreamTaskState prevState = streamTaskGetStatus(pTask);
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
int32_t code = streamTaskRestoreStatus(pTask); int32_t code = streamTaskRestoreStatus(pTask);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
char* pNew = streamTaskGetStatus(pTask)->name; char* pNew = streamTaskGetStatus(pTask).name;
int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num);
} else { } else {
@ -942,17 +960,24 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
return 0; return 0;
} }
SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) { int32_t streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) {
*pEpInfo = NULL;
int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
if (pInfo == NULL) {
return TSDB_CODE_FAILED;
}
if (pInfo->taskId == taskId) { if (pInfo->taskId == taskId) {
return pInfo; *pEpInfo = pInfo;
return TSDB_CODE_SUCCESS;
} }
} }
stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId); stError("s-task:%s failed to find upstream task:0x%x", pTask->id.idStr, taskId);
return NULL; return TSDB_CODE_FAILED;
} }
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) {
@ -973,10 +998,16 @@ SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) {
return NULL; return NULL;
} }
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId) {
char buf[128] = {0}; char buf[128] = {0};
sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId);
return taosStrdup(buf); *pId = taosStrdup(buf);
if (*pId == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} else {
return TSDB_CODE_SUCCESS;
}
} }
static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq) { static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
@ -1045,14 +1076,23 @@ int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo() { int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes) {
SActiveCheckpointInfo* pInfo = taosMemoryCalloc(1, sizeof(SActiveCheckpointInfo)); SActiveCheckpointInfo* pInfo = taosMemoryCalloc(1, sizeof(SActiveCheckpointInfo));
taosThreadMutexInit(&pInfo->lock, NULL); if (pInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = taosThreadMutexInit(&pInfo->lock, NULL);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo)); pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo));
pInfo->pReadyMsgList = taosArrayInit(4, sizeof(STaskCheckpointReadyInfo)); pInfo->pReadyMsgList = taosArrayInit(4, sizeof(STaskCheckpointReadyInfo));
pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo)); pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo));
return pInfo;
*pRes = pInfo;
return TSDB_CODE_SUCCESS;
} }
void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {

228
source/libs/stream/src/streamTaskSm.c
View File

@ -20,7 +20,17 @@
#include "ttimer.h" #include "ttimer.h"
#include "wal.h" #include "wal.h"
static int32_t initRes = 0;
#define GET_EVT_NAME(_ev) (StreamTaskEventList[(_ev)].name) #define GET_EVT_NAME(_ev) (StreamTaskEventList[(_ev)].name)
#define CHECK_RET_VAL(_exec) \
do { \
void* p = (_exec); \
if (p == NULL) { \
initRes = TSDB_CODE_OUT_OF_MEMORY; \
return; \
} \
} while (0);
SStreamTaskState StreamTaskStatusList[9] = { SStreamTaskState StreamTaskStatusList[9] = {
{.state = TASK_STATUS__READY, .name = "ready"}, {.state = TASK_STATUS__READY, .name = "ready"},
@ -67,16 +77,23 @@ static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus nex
static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; } static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; }
static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) { static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) {
char* p = streamTaskGetStatus(pTask)->name; char* p = streamTaskGetStatus(pTask).name;
stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p, stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p,
GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name); GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name);
SArray* pList = pTask->status.pSM->pWaitingEventList; SArray* pList = pTask->status.pSM->pWaitingEventList;
taosArrayPush(pList, pEvtInfo);
stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pList)); void* px = taosArrayPush(pList, pEvtInfo);
return 0; if (px == NULL) {
stError("s-task:%s failed to add into waiting list, total waiting events:%d, code: out of memory", pTask->id.idStr,
(int32_t)taosArrayGetSize(pList));
return TSDB_CODE_OUT_OF_MEMORY;
} else {
stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr,
(int32_t)taosArrayGetSize(pList));
return TSDB_CODE_SUCCESS;
}
} }
static int32_t stopTaskSuccFn(SStreamTask* pTask) { static int32_t stopTaskSuccFn(SStreamTask* pTask) {
@ -177,14 +194,14 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
GET_EVT_NAME(pEvtInfo->event), pSM->current.name); GET_EVT_NAME(pEvtInfo->event), pSM->current.name);
// remove it // remove it
taosArrayPop(pSM->pWaitingEventList); (void) taosArrayPop(pSM->pWaitingEventList);
STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event);
ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL);
pSM->pActiveTrans = pNextTrans; pSM->pActiveTrans = pNextTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
code = pNextTrans->pAction(pSM->pTask); code = pNextTrans->pAction(pSM->pTask);
if (pNextTrans->autoInvokeEndFn) { if (pNextTrans->autoInvokeEndFn) {
@ -193,7 +210,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
return code; return code;
} }
} else { } else {
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr,
pSM->current.name, GET_EVT_NAME(pEvtInfo->event), pSM->current.name, GET_EVT_NAME(pEvtInfo->event),
StreamTaskStatusList[pEvtInfo->status].name); StreamTaskStatusList[pEvtInfo->status].name);
@ -228,9 +245,9 @@ static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent eve
int32_t streamTaskRestoreStatus(SStreamTask* pTask) { int32_t streamTaskRestoreStatus(SStreamTask* pTask) {
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
int32_t code = 0; int32_t code = 0;
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) { if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) {
SStreamTaskState state = pSM->current; SStreamTaskState state = pSM->current;
@ -244,71 +261,78 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask) {
if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr,
pSM->prev.state.name, pSM->current.name); pSM->prev.state.name, pSM->current.name);
doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); code = doHandleWaitingEvent(pSM, "restore-pause/halt", pTask);
} else { } else {
stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name);
} }
} else { } else {
removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); (void)removeEventInWaitingList(pTask, TASK_EVENT_PAUSE); // ignore the return value,
code = -1; // failed to restore the status code = TSDB_CODE_FAILED; // failed to restore the status, since it is not in pause status
} }
taosThreadMutexUnlock(&pTask->lock); (void)taosThreadMutexUnlock(&pTask->lock);
return code; return code;
} }
SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) { int32_t streamCreateStateMachine(SStreamTask* pTask) {
initStateTransferTable(); int32_t code = initStateTransferTable();
if (code != TSDB_CODE_SUCCESS) {
return code;
}
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
SStreamTaskSM* pSM = taosMemoryCalloc(1, sizeof(SStreamTaskSM)); SStreamTaskSM* pSM = taosMemoryCalloc(1, sizeof(SStreamTaskSM));
if (pSM == NULL) { if (pSM == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM), stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM),
tstrerror(terrno)); tstrerror(terrno));
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSM->pTask = pTask; pSM->pTask = pTask;
pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo)); pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo));
if (pSM->pWaitingEventList == NULL) { if (pSM->pWaitingEventList == NULL) {
taosMemoryFree(pSM); taosMemoryFree(pSM);
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM), stError("s-task:%s failed to create task stateMachine, size:%d, code:%s", id, (int32_t)sizeof(SStreamTaskSM),
tstrerror(terrno)); tstrerror(terrno));
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
// set the initial state for the state-machine of stream task // set the initial state for the state-machine of stream task
pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
return pSM;
pTask->status.pSM = pSM;
return TSDB_CODE_SUCCESS;
} }
void* streamDestroyStateMachine(SStreamTaskSM* pSM) { void streamDestroyStateMachine(SStreamTaskSM* pSM) {
if (pSM == NULL) { if (pSM == NULL) {
return NULL; return;
} }
taosArrayDestroy(pSM->pWaitingEventList); taosArrayDestroy(pSM->pWaitingEventList);
taosMemoryFree(pSM); taosMemoryFree(pSM);
return NULL;
} }
static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans) { static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans) {
SStreamTask* pTask = pSM->pTask; SStreamTask* pTask = pSM->pTask;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t code = 0;
if (pTrans->attachEvent.event != 0) { if (pTrans->attachEvent.event != 0) {
attachWaitedEvent(pTask, &pTrans->attachEvent); code = attachWaitedEvent(pTask, &pTrans->attachEvent);
taosThreadMutexUnlock(&pTask->lock); if (code) {
return code;
}
(void) taosThreadMutexUnlock(&pTask->lock);
while (1) { while (1) {
// wait for the task to be here // wait for the task to be here
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
ETaskStatus s = streamTaskGetStatus(pTask)->state; ETaskStatus s = streamTaskGetStatus(pTask).state;
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already
stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event));
@ -325,42 +349,49 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
} else { // override current active trans } else { // override current active trans
pSM->pActiveTrans = pTrans; pSM->pActiveTrans = pTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
int32_t code = pTrans->pAction(pTask); code = pTrans->pAction(pTask);
// todo handle error code;
if (pTrans->autoInvokeEndFn) { if (pTrans->autoInvokeEndFn) {
streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL); int32_t c = streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL);
if (code == TSDB_CODE_SUCCESS) {
code = c;
}
} }
} }
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) { static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) {
SStreamTask* pTask = pSM->pTask; SStreamTask* pTask = pSM->pTask;
int32_t code = 0;
if (pTrans->attachEvent.event != 0) { if (pTrans->attachEvent.event != 0) {
SFutureHandleEventInfo info = pTrans->attachEvent; SFutureHandleEventInfo info = pTrans->attachEvent;
info.pParam = param; info.pParam = param;
info.callBackFn = callbackFn; info.callBackFn = callbackFn;
attachWaitedEvent(pTask, &info); code = attachWaitedEvent(pTask, &info);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
} else { // override current active trans } else { // override current active trans
pSM->pActiveTrans = pTrans; pSM->pActiveTrans = pTrans;
pSM->startTs = taosGetTimestampMs(); pSM->startTs = taosGetTimestampMs();
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
int32_t code = pTrans->pAction(pTask); code = pTrans->pAction(pTask);
// todo handle error code; // todo handle error code;
if (pTrans->autoInvokeEndFn) { if (pTrans->autoInvokeEndFn) {
streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param); int32_t c = streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param);
if (code == TSDB_CODE_SUCCESS) {
code = c;
}
} }
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) { int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
@ -369,11 +400,11 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
STaskStateTrans* pTrans = NULL; STaskStateTrans* pTrans = NULL;
while (1) { while (1) {
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
@ -383,7 +414,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
pTrans = streamTaskFindTransform(pSM->current.state, event); pTrans = streamTaskFindTransform(pSM->current.state, event);
if (pTrans == NULL) { if (pTrans == NULL) {
stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event)); stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event));
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -408,11 +439,11 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
STaskStateTrans* pTrans = NULL; STaskStateTrans* pTrans = NULL;
while (1) { while (1) {
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) { if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
@ -422,7 +453,7 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
pTrans = streamTaskFindTransform(pSM->current.state, event); pTrans = streamTaskFindTransform(pSM->current.state, event);
if (pTrans == NULL) { if (pTrans == NULL) {
stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name); stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -451,9 +482,10 @@ static void keepPrevInfo(SStreamTaskSM* pSM) {
int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) { int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) {
SStreamTask* pTask = pSM->pTask; SStreamTask* pTask = pSM->pTask;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t code = 0;
// do update the task status // do update the task status
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
STaskStateTrans* pTrans = pSM->pActiveTrans; STaskStateTrans* pTrans = pSM->pActiveTrans;
if (pTrans == NULL) { if (pTrans == NULL) {
@ -465,14 +497,14 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event), stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event),
pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); pSM->current.name, GET_EVT_NAME(pSM->prev.evt));
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
if (pTrans->event != event) { if (pTrans->event != event) {
stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event), stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event),
pSM->current.name, GET_EVT_NAME(pTrans->event)); pSM->current.name, GET_EVT_NAME(pTrans->event));
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
return TSDB_CODE_STREAM_INVALID_STATETRANS; return TSDB_CODE_STREAM_INVALID_STATETRANS;
} }
@ -482,38 +514,42 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
pSM->pActiveTrans = NULL; pSM->pActiveTrans = NULL;
// todo remove it // todo remove it
// todo: handle the error code
// on success callback, add into lock if necessary, or maybe we should add an option for this? // on success callback, add into lock if necessary, or maybe we should add an option for this?
pTrans->pSuccAction(pTask); code = pTrans->pSuccAction(pTask);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
// todo: add parameter to control lock // todo: add parameter to control lock
// after handling the callback function assigned by invoker, go on handling the waiting tasks // after handling the callback function assigned by invoker, go on handling the waiting tasks
if (callbackFn != NULL) { if (callbackFn != NULL) {
stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event));
callbackFn(pSM->pTask, param); int32_t ret = callbackFn(pSM->pTask, param);
if (ret != TSDB_CODE_SUCCESS) {
// todo handle error
}
stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event)); stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event));
} }
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
// tasks in waiting list // tasks in waiting list
if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); code = doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask);
} else { } else {
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
int64_t el = (taosGetTimestampMs() - pSM->startTs); int64_t el = (taosGetTimestampMs() - pSM->startTs);
stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id, stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id,
GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name);
} }
return TSDB_CODE_SUCCESS; return code;
} }
SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask) { SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask) {
return &pTask->status.pSM->current; // copy one obj in case of multi-thread environment return pTask->status.pSM->current; // copy one obj in case of multi-thread environment
} }
ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask) { ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask) {
@ -527,14 +563,14 @@ const char* streamTaskGetStatusStr(ETaskStatus status) {
void streamTaskResetStatus(SStreamTask* pTask) { void streamTaskResetStatus(SStreamTask* pTask) {
SStreamTaskSM* pSM = pTask->status.pSM; SStreamTaskSM* pSM = pTask->status.pSM;
taosThreadMutexLock(&pTask->lock); (void) taosThreadMutexLock(&pTask->lock);
stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr, stDebug("s-task:%s level:%d fill-history:%d vgId:%d set uninit, prev status:%s", pTask->id.idStr,
pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name); pTask->info.taskLevel, pTask->info.fillHistory, pTask->pMeta->vgId, pSM->current.name);
pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT]; pSM->current = StreamTaskStatusList[TASK_STATUS__UNINIT];
pSM->pActiveTrans = NULL; pSM->pActiveTrans = NULL;
taosArrayClear(pSM->pWaitingEventList); taosArrayClear(pSM->pWaitingEventList);
taosThreadMutexUnlock(&pTask->lock); (void) taosThreadMutexUnlock(&pTask->lock);
// clear the downstream ready status // clear the downstream ready status
pTask->status.downstreamReady = 0; pTask->status.downstreamReady = 0;
@ -577,8 +613,7 @@ STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStr
} }
int32_t initStateTransferTable() { int32_t initStateTransferTable() {
taosThreadOnce(&streamTaskStateMachineInit, doInitStateTransferTable); return taosThreadOnce(&streamTaskStateMachineInit, doInitStateTransferTable);
return TSDB_CODE_SUCCESS;
} }
//clang-format off //clang-format off
@ -587,92 +622,93 @@ void doInitStateTransferTable(void) {
// initialization event handle // initialization event handle
STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL); STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanHistoryTaskReady, NULL); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanHistoryTaskReady, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
// scan-history related event // scan-history related event
trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
// halt stream task, from other task status // halt stream task, from other task status
trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT}; SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT};
trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
// checkpoint related event // checkpoint related event
trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
// pause & resume related event handle // pause & resume related event handle
trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE}; info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE};
trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
// resume is completed by restore status of state-machine // resume is completed by restore status of state-machine
// stop related event // stop related event
trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, stopTaskSuccFn, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
// dropping related event // dropping related event
trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
taosArrayPush(streamTaskSMTrans, &trans); CHECK_RET_VAL(taosArrayPush(streamTaskSMTrans, &trans));
} }
//clang-format on //clang-format on

11
source/libs/stream/src/streamTimer.c
View File

@ -38,3 +38,14 @@ void streamTimerCleanUp() {
tmr_h streamTimerGetInstance() { tmr_h streamTimerGetInstance() {
return streamTimer; return streamTimer;
} }
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg) {
while (1) {
bool ret = taosTmrReset(fp, mseconds, param, handle, pTmrId);
if (ret) {
break;
}
stError("vgId:%d failed to reset %s, try again", vgId, pMsg);
}
}

4
source/libs/stream/test/backendTest.cpp
View File

@ -43,7 +43,9 @@ SStreamState *stateCreate(const char *path) {
pTask->ver = 1024; pTask->ver = 1024;
pTask->id.streamId = 1023; pTask->id.streamId = 1023;
pTask->id.taskId = 1111111; pTask->id.taskId = 1111111;
SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL); SStreamMeta *pMeta = NULL;
int32_t code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pMeta);
pTask->pMeta = pMeta; pTask->pMeta = pMeta;
SStreamState *p = streamStateOpen((char *)path, pTask, 0, 0); SStreamState *p = streamStateOpen((char *)path, pTask, 0, 0);

5
source/libs/wal/src/walRef.c
View File

@ -52,15 +52,14 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) {
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) { if (ver < pWal->vers.firstVer || ver > pWal->vers.lastVer) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
terrno = TSDB_CODE_WAL_INVALID_VER; return TSDB_CODE_WAL_INVALID_VER;
return -1;
} }
pRef->refVer = ver; pRef->refVer = ver;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
} }
return 0; return TSDB_CODE_SUCCESS;
} }
void walRefFirstVer(SWal *pWal, SWalRef *pRef) { void walRefFirstVer(SWal *pWal, SWalRef *pRef) {

29
source/util/src/thash.c
View File

@ -316,7 +316,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) {
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) { int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const void *data, size_t size) {
if (pHashObj == NULL || key == NULL || keyLen == 0) { if (pHashObj == NULL || key == NULL || keyLen == 0) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
return -1; return TSDB_CODE_INVALID_PTR;
} }
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
@ -331,6 +331,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
// disable resize // disable resize
taosHashRLock(pHashObj); taosHashRLock(pHashObj);
int32_t code = TSDB_CODE_SUCCESS;
uint32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); uint32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHashEntry *pe = pHashObj->hashList[slot]; SHashEntry *pe = pHashObj->hashList[slot];
@ -352,36 +353,34 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo
// no data in hash table with the specified key, add it into hash table // no data in hash table with the specified key, add it into hash table
SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal);
if (pNewNode == NULL) { if (pNewNode == NULL) {
return -1; terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
} }
pushfrontNodeInEntryList(pe, pNewNode); pushfrontNodeInEntryList(pe, pNewNode);
taosHashEntryWUnlock(pHashObj, pe);
// enable resize
taosHashRUnlock(pHashObj);
atomic_add_fetch_64(&pHashObj->size, 1); atomic_add_fetch_64(&pHashObj->size, 1);
return 0;
} else { } else {
// not support the update operation, return error // not support the update operation, return error
if (pHashObj->enableUpdate) { if (pHashObj->enableUpdate) {
SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal);
if (pNewNode == NULL) { if (pNewNode == NULL) {
return -1; terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
} }
doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode);
} else { } else {
terrno = TSDB_CODE_DUP_KEY; terrno = TSDB_CODE_DUP_KEY;
code = terrno;
goto _exit;
} }
taosHashEntryWUnlock(pHashObj, pe);
// enable resize
taosHashRUnlock(pHashObj);
return pHashObj->enableUpdate ? 0 : -2;
} }
_exit:
taosHashEntryWUnlock(pHashObj, pe);
taosHashRUnlock(pHashObj);
return code;
} }
static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef); static void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void **d, int32_t *size, bool addRef);

3
tests/system-test/7-tmq/tmqClientConsLog.py
View File

@ -16,6 +16,9 @@ sys.path.append("./7-tmq")
from tmqCommon import * from tmqCommon import *
class TDTestCase: class TDTestCase:
clientCfgDict = {'debugFlag': 135}
updatecfgDict = {'debugFlag': 131, 'clientCfg':clientCfgDict}
def __init__(self): def __init__(self):
self.vgroups = 3 self.vgroups = 3
self.ctbNum = 10 self.ctbNum = 10