xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: return error code

This commit is contained in:
kailixu 2024-07-22 15:53:20 +08:00
parent b337fa628b 5328c6dbea
commit cdc85d9588
307 changed files with 17001 additions and 12625 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Jenkinsfile2
View File

@ -401,7 +401,7 @@ pipeline {
} }
} }
stage('linux test') { stage('linux test') {
agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() } options { skipDefaultCheckout() }
when { when {
changeRequest() changeRequest()
@ -416,7 +416,7 @@ pipeline {
echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
} }
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 150, unit: 'MINUTES'){ timeout(time: 200, unit: 'MINUTES'){
pre_test() pre_test()
script { script {
sh ''' sh '''
@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2 export DEFAULT_RETRY_TIME=2
date date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + ''' ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 1200 ''' + extra_param + '''
''' '''
} }
} }

2
docs/en/12-taos-sql/12-distinguished.md
View File

@ -38,7 +38,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## Windowed Queries ## Windowed Queries
Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are four kinds of windows: time window, status window, session window, and event window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows: Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are five kinds of windows: time window, status window, session window, event window, and count window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows:
```sql ```sql
window_clause: { window_clause: {

2
docs/zh/12-taos-sql/12-distinguished.md
View File

@ -39,7 +39,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## 窗口切分查询 ## 窗口切分查询
TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）四种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）、计数窗口（count window）五种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。
窗口子句语法如下： 窗口子句语法如下：

2
include/common/rsync.h
View File

@ -12,7 +12,7 @@ extern "C" {
#include "tarray.h" #include "tarray.h"
void stopRsync(); void stopRsync();
void startRsync(); int32_t startRsync();
int32_t uploadByRsync(const char* id, const char* path); int32_t uploadByRsync(const char* id, const char* path);
int32_t downloadRsync(const char* id, const char* path); int32_t downloadRsync(const char* id, const char* path);
int32_t deleteRsync(const char* id); int32_t deleteRsync(const char* id);

2
include/common/tdatablock.h
View File

@ -253,7 +253,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc); int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);
SSDataBlock* createDataBlock(); SSDataBlock* createDataBlock();
void* blockDataDestroy(SSDataBlock* pBlock); void blockDataDestroy(SSDataBlock* pBlock);
void blockDataFreeRes(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createSpecialDataBlock(EStreamType type); SSDataBlock* createSpecialDataBlock(EStreamType type);

1
include/common/tmsg.h
View File

@ -3612,6 +3612,7 @@ typedef struct {
int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset);
int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset);
void tDeleteSTqOffset(void* val);
typedef struct SMqVgOffset { typedef struct SMqVgOffset {
int64_t consumerId; int64_t consumerId;

60
include/libs/executor/storageapi.h
View File

@ -229,15 +229,15 @@ typedef struct SStoreTqReader {
bool (*tqReaderCurrentBlockConsumed)(); bool (*tqReaderCurrentBlockConsumed)();
struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it
// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it // int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
int32_t (*tqReaderSetSubmitMsg)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it
// bool (*tqReaderNextBlockFilterOut)(); // bool (*tqReaderNextBlockFilterOut)();
} SStoreTqReader; } SStoreTqReader;
typedef struct SStoreSnapshotFn { typedef struct SStoreSnapshotFn {
bool (*taosXGetTablePrimaryKey)(SSnapContext *ctx); bool (*taosXGetTablePrimaryKey)(SSnapContext* ctx);
void (*taosXSetTablePrimaryKey)(SSnapContext *ctx, int64_t uid); void (*taosXSetTablePrimaryKey)(SSnapContext* ctx, int64_t uid);
int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid); int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid);
int32_t (*destroySnapshot)(SSnapContext* ctx); int32_t (*destroySnapshot)(SSnapContext* ctx);
SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx);
@ -270,7 +270,7 @@ typedef struct SStoreMeta {
int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio); int32_t payloadLen, double selectivityRatio);
void* (*storeGetIndexInfo)(void *pVnode); void* (*storeGetIndexInfo)(void* pVnode);
void* (*getInvertIndex)(void* pVnode); void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list); int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
@ -329,29 +329,31 @@ typedef struct {
typedef struct SStateStore { typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname); int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used); int32_t* pWinCode);
int32_t (*streamStateClearBuff)(SStreamState* pState, void* pVal); void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
void (*streamStateClearBuff)(SStreamState* pState, void* pVal);
void (*streamStateFreeVal)(void* val); void (*streamStateFreeVal)(void* val);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); void (*streamStateDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateClear)(SStreamState* pState); void (*streamStateClear)(SStreamState* pState);
void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex); void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex);
int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key); void (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); void (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur);
int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); void (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur);
SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key);
SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key);
@ -363,26 +365,29 @@ typedef struct SStateStore {
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen); int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen,
int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); int32_t* pWinCode);
int32_t (*streamStateSessionReset)(SStreamState* pState, void* pVal); void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
int32_t (*streamStateSessionClear)(SStreamState* pState); void (*streamStateSessionReset)(SStreamState* pState, void* pVal);
void (*streamStateSessionClear)(SStreamState* pState);
int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur, int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen); const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen); void** ppVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType,
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); int32_t pkLen, SUpdateInfo** ppInfo);
int32_t (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol,
int32_t primaryKeyCol, TSKEY* pMaxResTs);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
@ -391,7 +396,8 @@ typedef struct SStateStore {
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
@ -413,7 +419,7 @@ typedef struct SStateStore {
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId); SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove); void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState); int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState); void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove); void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts); void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);

2
include/libs/function/function.h
View File

@ -294,7 +294,7 @@ typedef struct SPoint {
void *val; void *val;
} SPoint; } SPoint;
int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
int32_t inputType); int32_t inputType);
#define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25 #define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25

3
include/libs/qcom/query.h
View File

@ -192,6 +192,7 @@ typedef struct SBoundColInfo {
int16_t* pColIndex; // bound index => schema index int16_t* pColIndex; // bound index => schema index
int32_t numOfCols; int32_t numOfCols;
int32_t numOfBound; int32_t numOfBound;
bool hasBoundCols;
} SBoundColInfo; } SBoundColInfo;
typedef struct STableColsData { typedef struct STableColsData {
@ -334,7 +335,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
void destroyQueryExecRes(SExecResult* pRes); void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
char* parseTagDatatoJson(void* p); void parseTagDatatoJson(void* p, char** jsonStr);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);

5
include/libs/scalar/filter.h
View File

@ -58,12 +58,13 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict
extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info); extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows); extern int32_t filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows,
bool *keep);
/* condition split interface */ /* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,
SNode **pOtherCond); SNode **pOtherCond);
bool filterIsMultiTableColsCond(SNode *pCond); int32_t filterIsMultiTableColsCond(SNode *pCond, bool *res);
EConditionType filterClassifyCondition(SNode *pNode); EConditionType filterClassifyCondition(SNode *pNode);
#ifdef __cplusplus #ifdef __cplusplus

40
include/libs/stream/streamState.h
View File

@ -32,7 +32,7 @@ extern "C" {
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId); SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove); void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState); void streamStateCommit(SStreamState* pState);
void streamStateDestroy(SStreamState* pState, bool remove); void streamStateDestroy(SStreamState* pState, bool remove);
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark); int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
int32_t streamStateDelTaskDb(SStreamState* pState); int32_t streamStateDelTaskDb(SStreamState* pState);
@ -41,22 +41,23 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void*
int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool streamStateCheck(SStreamState* pState, const SWinKey* key); bool streamStateCheck(SStreamState* pState, const SWinKey* key);
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal); int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal);
int32_t streamStateDel(SStreamState* pState, const SWinKey* key); void streamStateDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateClear(SStreamState* pState); void streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex); void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex);
int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
// session window // session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen); int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); void streamStateSessionDel(SStreamState* pState, const SSessionKey* key);
int32_t streamStateSessionReset(SStreamState* pState, void* pVal); void streamStateSessionReset(SStreamState* pState, void* pVal);
int32_t streamStateSessionClear(SStreamState* pState); void streamStateSessionClear(SStreamState* pState);
int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
@ -70,21 +71,22 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
// state window // state window
int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
// fill // fill
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); void streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used); int32_t* pWinCode);
int32_t streamStateClearBuff(SStreamState* pState, void* pVal); void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
void streamStateClearBuff(SStreamState* pState, void* pVal);
void streamStateFreeVal(void* val); void streamStateFreeVal(void* val);
// count window // count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen); int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
@ -97,11 +99,11 @@ void streamStateResetCur(SStreamStateCur* pCur);
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts); void streamStateReloadInfo(SStreamState* pState, TSKEY ts);

46
include/libs/stream/tstream.h
View File

@ -59,8 +59,6 @@ extern "C" {
#define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_STOP_ALL_TASKS (-5)
#define STREAM_EXEC_T_RESUME_TASK (-6) #define STREAM_EXEC_T_RESUME_TASK (-6)
#define STREAM_EXEC_T_ADD_FAILED_TASK (-7) #define STREAM_EXEC_T_ADD_FAILED_TASK (-7)
// the load and start stream task should be executed after snode has started successfully, since the load of stream
// tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
typedef struct SStreamTask SStreamTask; typedef struct SStreamTask SStreamTask;
typedef struct SStreamQueue SStreamQueue; typedef struct SStreamQueue SStreamQueue;
@ -190,7 +188,7 @@ typedef struct {
SSDataBlock* pBlock; SSDataBlock* pBlock;
} SStreamRefDataBlock; } SStreamRefDataBlock;
SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit);
void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
typedef struct { typedef struct {
@ -530,8 +528,8 @@ typedef struct STaskUpdateEntry {
typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5); SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** pTask);
void tFreeStreamTask(SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
@ -631,8 +629,10 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo);
#if 0
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
#endif
void streamTaskInputFail(SStreamTask* pTask); void streamTaskInputFail(SStreamTask* pTask);
@ -641,8 +641,8 @@ bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId);
SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask); SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask);
const char* streamTaskGetStatusStr(ETaskStatus status); const char* streamTaskGetStatusStr(ETaskStatus status);
void streamTaskResetStatus(SStreamTask* pTask); void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask);
@ -659,10 +659,10 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt); int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
int32_t streamExecTask(SStreamTask* pTask); int32_t streamExecTask(SStreamTask* pTask);
int32_t streamResumeTask(SStreamTask* pTask); void streamResumeTask(SStreamTask* pTask);
int32_t streamTrySchedExec(SStreamTask* pTask); int32_t streamTrySchedExec(SStreamTask* pTask);
int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType); int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType);
int32_t streamTaskResumeInFuture(SStreamTask* pTask); void streamTaskResumeInFuture(SStreamTask* pTask);
void streamTaskClearSchedIdleInfo(SStreamTask* pTask); void streamTaskClearSchedIdleInfo(SStreamTask* pTask);
void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime); void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime);
@ -675,8 +675,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp);
// check downstream status // check downstream status
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask); void streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id); void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id);
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo); void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo);
// fill-history task // fill-history task
@ -726,18 +726,18 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta // stream task meta
void streamMetaInit(); void streamMetaInit();
void streamMetaCleanup(); void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, int32_t vgId,
int32_t vgId, int64_t stage, startComplete_fn_t fn); int64_t stage, startComplete_fn_t fn, SStreamMeta** pMeta);
void streamMetaClose(SStreamMeta* streamMeta); void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
SStreamTask* streamMetaAcquireOneTask(SStreamTask* pTask); void streamMetaAcquireOneTask(SStreamTask* pTask);
void streamMetaClear(SStreamMeta* pMeta); void streamMetaClear(SStreamMeta* pMeta);
void streamMetaInitBackend(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta);
@ -760,7 +760,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta);
void streamMetaWUnLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta);
void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId); void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId);
SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pTaskList);
void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader);
void streamMetaLoadAllTasks(SStreamMeta* pMeta); void streamMetaLoadAllTasks(SStreamMeta* pMeta);
int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); int32_t streamMetaStartAllTasks(SStreamMeta* pMeta);
@ -771,6 +771,8 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer // timer
tmr_h streamTimerGetInstance(); tmr_h streamTimerGetInstance();
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
@ -787,11 +789,11 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
int32_t setCode); int32_t setCode);
int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask); int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq); int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq);
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo(); int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes);
// stream task state machine, and event handling // stream task state machine, and event handling
SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); int32_t streamCreateStateMachine(SStreamTask* pTask);
void* streamDestroyStateMachine(SStreamTaskSM* pSM); void streamDestroyStateMachine(SStreamTaskSM* pSM);
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event); int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn,
void* param); void* param);
@ -805,7 +807,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp); void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
#ifdef __cplusplus #ifdef __cplusplus

49
include/libs/stream/tstreamFileState.h
View File

@ -18,10 +18,10 @@
#include "os.h" #include "os.h"
#include "storageapi.h"
#include "tarray.h" #include "tarray.h"
#include "tdef.h" #include "tdef.h"
#include "tlist.h" #include "tlist.h"
#include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -32,7 +32,7 @@ typedef SList SStreamSnapshot;
typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen); typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef int32_t (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos); typedef void (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos);
typedef void (*_state_buff_cleanup_fn)(void* pRowBuff); typedef void (*_state_buff_cleanup_fn)(void* pRowBuff);
typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num); typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num);
@ -40,28 +40,30 @@ typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const voi
typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen); typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen);
typedef int32_t (*_state_file_clear_fn)(SStreamState* pState); typedef int32_t (*_state_file_clear_fn)(SStreamState* pState);
typedef int32_t (*_state_fun_get_fn) (SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2); typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2);
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId,
int64_t checkpointId, int8_t type); int8_t type);
void streamFileStateDestroy(SStreamFileState* pFileState); void streamFileStateDestroy(SStreamFileState* pFileState);
void streamFileStateClear(SStreamFileState* pFileState); void streamFileStateClear(SStreamFileState* pFileState);
bool needClearDiskBuff(SStreamFileState* pFileState); bool needClearDiskBuff(SStreamFileState* pFileState);
void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used); void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used);
int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen); int32_t* pWinCode);
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal); int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen); bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen);
void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState); SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState); void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId); void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list); int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list);
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark); int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark);
@ -76,16 +78,18 @@ SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState); int32_t getRowStateRowSize(SStreamFileState* pFileState);
// session window // session window
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen); int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen);
int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur,
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen);
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); void recoverSesssion(SStreamFileState* pFileState, int64_t ckId);
void sessionWinStateClear(SStreamFileState* pFileState); void sessionWinStateClear(SStreamFileState* pFileState);
void sessionWinStateCleanup(void* pBuff); void sessionWinStateCleanup(void* pBuff);
@ -95,19 +99,22 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState,
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey); SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count); SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur); void sessionWinStateMoveToNext(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn); int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn);
// state window // state window
int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
// count window // count window
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen); int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
//function // function
int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen); int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
#ifdef __cplusplus #ifdef __cplusplus

9
include/libs/stream/tstreamUpdate.h
View File

@ -25,9 +25,12 @@
extern "C" { extern "C" {
#endif #endif
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo); SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol,
TSKEY* pMaxResTs);
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid); bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo* pInfo); void updateInfoDestroy(SUpdateInfo* pInfo);

2
include/os/osDef.h
View File

@ -220,7 +220,7 @@ void syslog(int unused, const char *format, ...);
// Linux, length of name must <= 16 (the last '\0' included) // Linux, length of name must <= 16 (the last '\0' included)
#define setThreadName(name) \ #define setThreadName(name) \
do { \ do { \
prctl(PR_SET_NAME, (name)); \ (void)prctl(PR_SET_NAME, (name)); \
} while (0) } while (0)
#define getThreadName(name) \ #define getThreadName(name) \
do { \ do { \

1
include/os/osMemory.h
View File

@ -55,6 +55,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n))
#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) #define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n))
#define TAOS_MEMMOVE(_d, _s, _n) ((void)memmove(_d, _s, _n))
#define taosMemoryFreeClear(ptr) \ #define taosMemoryFreeClear(ptr) \
do { \ do { \

1
include/os/osString.h
View File

@ -60,6 +60,7 @@ typedef enum { M2C = 0, C2M } ConvType;
} while (0) } while (0)
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
char *tstrdup(const char *src); char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosUcs4len(TdUcs4 *ucs4);

5
include/util/taoserror.h
View File

@ -53,7 +53,7 @@ int32_t taosGetErrSize();
#define terrln (*taosGetErrln()) #define terrln (*taosGetErrln())
#define SET_ERROR_MSG(MSG, ...) \ #define SET_ERROR_MSG(MSG, ...) \
snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__) (void)snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
#define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
@ -224,6 +224,7 @@ int32_t taosGetErrSize();
// #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x // #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x
#define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315) #define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315)
#define TSDB_CODE_MND_INVALID_PLATFORM TAOS_DEF_ERROR_CODE(0, 0x0316) #define TSDB_CODE_MND_INVALID_PLATFORM TAOS_DEF_ERROR_CODE(0, 0x0316)
#define TSDB_CODE_MND_RETURN_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x0317)
// mnode-sdb // mnode-sdb
#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal #define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal
@ -555,6 +556,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D) #define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D)
#define TSDB_CODE_TDB_INCONSISTENT_DB_ID TAOS_DEF_ERROR_CODE(0, 0x061E)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)
@ -596,6 +598,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731) #define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732) #define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733) #define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
#define TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE TAOS_DEF_ERROR_CODE(0, 0x0734)
// grant // grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)

3
include/util/thash.h
View File

@ -28,8 +28,7 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *);
#define HASH_KEY_ALREADY_EXISTS (-2) #define HASH_NODE_EXIST(code) (code == TSDB_CODE_DUP_KEY)
#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
/** /**
* murmur hash algorithm * murmur hash algorithm

2
include/util/tlist.h
View File

@ -224,7 +224,7 @@ void tdListInit(SList *list, int32_t eleSize);
void tdListEmpty(SList *list); void tdListEmpty(SList *list);
SList *tdListNew(int32_t eleSize); SList *tdListNew(int32_t eleSize);
void *tdListFree(SList *list); void *tdListFree(SList *list);
void *tdListFreeP(SList *list, FDelete fp); void tdListFreeP(SList *list, FDelete fp);
void tdListPrependNode(SList *list, SListNode *node); void tdListPrependNode(SList *list, SListNode *node);
void tdListAppendNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node);
int32_t tdListPrepend(SList *list, void *data); int32_t tdListPrepend(SList *list, void *data);

2
include/util/tscalablebf.h
View File

@ -34,7 +34,7 @@ typedef struct SScalableBf {
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf); int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len); int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes);
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf* pSBf); void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder); int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);

1
packaging/deb/DEBIAN/postinst
View File

@ -8,6 +8,7 @@ if command -v sudo > /dev/null; then
csudo="sudo " csudo="sudo "
fi fi
${csudo}mkdir -p ${insmetaPath}
${csudo}chmod -R 744 ${insmetaPath} ${csudo}chmod -R 744 ${insmetaPath}
cd ${insmetaPath} cd ${insmetaPath}
${csudo}./post.sh ${csudo}./post.sh

1
packaging/deb/DEBIAN/prerm
View File

@ -33,6 +33,7 @@ else
${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || :

13
packaging/deb/makedeb.sh
View File

@ -76,11 +76,11 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || :
fi fi
if [ -f "%{_compiledir}/../../../explorer/target/taos-explorer.service" ]; then if [ -f "${compile_dir}/../../../explorer/target/taos-explorer.service" ]; then
cp %{_compiledir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || : cp ${compile_dir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || :
fi fi
if [ -f "%{_compiledir}/../../../explorer/server/example/explorer.toml" ]; then if [ -f "${compile_dir}/../../../explorer/server/example/explorer.toml" ]; then
cp %{_compiledir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || : cp ${compile_dir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || :
fi fi
cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin
@ -90,6 +90,11 @@ cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin
cp ${top_dir}/../enterprise/packaging/start-all.sh ${pkg_dir}${install_home_path}/bin
cp ${top_dir}/../enterprise/packaging/stop-all.sh ${pkg_dir}${install_home_path}/bin
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/start-all.sh
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/stop-all.sh
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin

4
packaging/rpm/makerpm.sh
View File

@ -70,8 +70,8 @@ else
fi fi
cd ${top_dir} cd ${top_dir}
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper -t ver-${tdengine_ver}"
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper -t ver-${tdengine_ver}`
echo "taoskeeper_binary: ${taoskeeper_binary}" echo "taoskeeper_binary: ${taoskeeper_binary}"
cd ${package_dir} cd ${package_dir}

8
packaging/rpm/tdengine.spec
View File

@ -56,6 +56,10 @@ mkdir -p %{buildroot}%{homepath}/include
#mkdir -p %{buildroot}%{homepath}/init.d #mkdir -p %{buildroot}%{homepath}/init.d
mkdir -p %{buildroot}%{homepath}/script mkdir -p %{buildroot}%{homepath}/script
if [ -f %{_compiledir}/../packaging/cfg/taosd.service ]; then
cp %{_compiledir}/../packaging/cfg/taosd.service %{buildroot}%{homepath}/cfg
fi
cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg
if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then
cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg
@ -91,6 +95,10 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/../../enterprise/packaging/start-all.sh %{buildroot}%{homepath}/bin
cp %{_compiledir}/../../enterprise/packaging/stop-all.sh %{buildroot}%{homepath}/bin
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/start-all.sh
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/stop-all.sh
if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then
cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin

51
packaging/tools/install.sh
View File

@ -156,7 +156,7 @@ done
#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh) tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh start-all.sh stop-all.sh)
if [ "${verMode}" == "cluster" ]; then if [ "${verMode}" == "cluster" ]; then
services=(${serverName} ${adapterName} ${xname} ${explorerName} ${keeperName}) services=(${serverName} ${adapterName} ${xname} ${explorerName} ${keeperName})
elif [ "${verMode}" == "edge" ]; then elif [ "${verMode}" == "edge" ]; then
@ -221,6 +221,8 @@ function install_bin() {
${csudo}cp -r ${script_dir}/bin/remove.sh ${install_main_dir}/bin ${csudo}cp -r ${script_dir}/bin/remove.sh ${install_main_dir}/bin
else else
${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin
${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/bin
${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/bin
fi fi
if [[ "${verMode}" == "cluster" && "${verType}" != "client" ]]; then if [[ "${verMode}" == "cluster" && "${verType}" != "client" ]]; then
@ -576,7 +578,9 @@ function install_taosd_config() {
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile} ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile}
${csudo}echo "monitor 1" >>${script_dir}/cfg/${configFile} ${csudo}echo "monitor 1" >>${script_dir}/cfg/${configFile}
${csudo}echo "monitorFQDN ${serverFqdn}" >>${script_dir}/cfg/${configFile} ${csudo}echo "monitorFQDN ${serverFqdn}" >>${script_dir}/cfg/${configFile}
if [ "$verMode" == "cluster" ]; then
${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile} ${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile}
fi
if [ -f "${configDir}/${configFile}" ]; then if [ -f "${configDir}/${configFile}" ]; then
${csudo}cp ${fileName} ${configDir}/${configFile}.new ${csudo}cp ${fileName} ${configDir}/${configFile}.new
@ -594,6 +598,7 @@ function install_config() {
[ ! -z $1 ] && return 0 || : # only install client [ ! -z $1 ] && return 0 || : # only install client
if ((${update_flag} == 1)); then if ((${update_flag} == 1)); then
install_taosd_config
return 0 return 0
fi fi
@ -652,8 +657,6 @@ function install_data() {
function install_connector() { function install_connector() {
if [ -d "${script_dir}/connector/" ]; then if [ -d "${script_dir}/connector/" ]; then
${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector"
${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/ || echo "failed to copy start-all.sh"
${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/ || echo "failed to copy stop-all.sh"
${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md" ${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md"
fi fi
} }
@ -911,38 +914,35 @@ function updateProduct() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}"
[ -f ${configDir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] && [ -f ${configDir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] &&
echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml" echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml"
if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml" echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml"
fi
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}"
else else
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ./${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ./${serverName}${NC}"
[ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}"
fi fi
echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}" echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
fi fi
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo echo
echo "${productName} is updated successfully!" echo "${productName} is updated successfully!"
echo echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo start all the components : ./start-all.sh${NC}"
fi
echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}"
echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}" echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}"
fi fi
else else
@ -1007,39 +1007,36 @@ function installProduct() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}"
[ -f ${configDir}/${clientName}adapter.toml ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${configDir}/${clientName}adapter.toml ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To configure ${clientName}Adapter ${NC}\t: edit ${configDir}/${clientName}adapter.toml" echo -e "${GREEN_DARK}To configure ${clientName}Adapter ${NC}\t: edit ${configDir}/${clientName}adapter.toml"
if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml" echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml"
fi
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}"
else else
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${serverName}${NC}"
[ -f ${installDir}/bin/${clientName}adapter ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}" echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}"
fi fi
echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}" echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}" echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
fi fi
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo echo
echo "${productName} is installed successfully!" echo "${productName} is installed successfully!"
echo echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo start all the components : sudo ./start-all.sh${NC}"
fi
echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}"
echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}" echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}"
fi fi
echo echo

2
packaging/tools/makepkg.sh
View File

@ -232,7 +232,7 @@ fi
if [ "$verMode" == "cluster" ]; then if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" remove_temp.sh sed -i "s/PREFIX=\"taos\"/PREFIX=\"${clientName2}\"/g" remove_temp.sh
sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh mv remove_temp.sh ${install_dir}/bin/remove.sh
fi fi

123
packaging/tools/post.sh
View File

@ -104,6 +104,8 @@ fi
function log_print(){ function log_print(){
now=$(date +"%D %T") now=$(date +"%D %T")
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
${csudo} touch ${install_log_path}
echo "$now $1" >> ${install_log_path} echo "$now $1" >> ${install_log_path}
} }
@ -235,11 +237,14 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/rmtaos || : ${csudo}rm -f ${bin_link_dir}/rmtaos || :
${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/*explorer || : ${csudo}rm -f ${bin_link_dir}/*explorer || :
${csudo}rm -f ${bin_link_dir}/start-all.sh || :
${csudo}rm -f ${bin_link_dir}/stop-all.sh || :
${csudo}chmod 0555 ${bin_dir}/* ${csudo}chmod 0555 ${bin_dir}/*
@ -275,8 +280,14 @@ function install_bin() {
if [ -x ${bin_dir}/taoskeeper ]; then if [ -x ${bin_dir}/taoskeeper ]; then
${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1 ${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1
fi fi
if [ -x ${bin_dir}/*explorer ]; then if [ -x ${bin_dir}/taos-explorer ]; then
${csudo}ln -s ${bin_dir}/*explorer ${bin_link_dir}/*explorer 2>>${install_log_path} || return 1 ${csudo}ln -s ${bin_dir}/taos-explorer ${bin_link_dir}/taos-explorer 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/start-all.sh ]; then
${csudo}ln -s ${bin_dir}/start-all.sh ${bin_link_dir}/start-all.sh 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/stop-all.sh ]; then
${csudo}ln -s ${bin_dir}/stop-all.sh ${bin_link_dir}/stop-all.sh 2>>${install_log_path} || return 1
fi fi
log_print "install bin success" log_print "install bin success"
} }
@ -474,6 +485,22 @@ function install_taoskeeper_config() {
${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir} ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir}
} }
function install_taos-explorer_config() {
if [ ! -f "${cfg_install_dir}/explorer.toml" ]; then
[ ! -d ${cfg_install_dir} ] &&
${csudo}${csudo}mkdir -p ${cfg_install_dir}
[ -f ${cfg_dir}/explorer.toml ] && ${csudo}cp ${cfg_dir}/explorer.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/explorer.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/explorer.toml
fi
[ -f ${cfg_dir}/explorer.toml ] &&
${csudo}mv ${cfg_dir}/explorer.toml ${cfg_dir}/explorer.toml.new
[ -f ${cfg_install_dir}/explorer.toml ] &&
${csudo}ln -s ${cfg_install_dir}/explorer.toml ${cfg_dir}
}
function install_config() { function install_config() {
log_print "start install config from ${cfg_dir} to ${cfg_install_dir}" log_print "start install config from ${cfg_dir} to ${cfg_install_dir}"
if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then
@ -610,29 +637,11 @@ function clean_service_on_systemd() {
function install_service_on_systemd() { function install_service_on_systemd() {
clean_service_on_systemd clean_service_on_systemd
taosd_service_config="${service_config_dir}/taosd.service" [ -f ${script_dir}/../cfg/taosd.service ] &&\
${csudo}cp ${script_dir}/../cfg/taosd.service \
${service_config_dir}/ || :
${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}" ${csudo}systemctl daemon-reload
${csudo}bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}"
${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}"
${csudo}bash -c "echo >> ${taosd_service_config}"
${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}"
${csudo}bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}"
${csudo}bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}"
${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}"
${csudo}bash -c "echo >> ${taosd_service_config}"
${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}"
${csudo}systemctl enable taosd ${csudo}systemctl enable taosd
} }
@ -652,6 +661,11 @@ function install_service_on_launchctl() {
${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : ${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
fi fi
if [ -f ${install_main_dir}/service/com.taosdata.taos-explorer.plist ]; then
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist > /dev/null 2>&1 || :
${csudo}cp ${install_main_dir}/service/com.taosdata.taos-explorer.plist /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || :
fi
} }
function install_taosadapter_service() { function install_taosadapter_service() {
@ -672,6 +686,15 @@ function install_taoskeeper_service() {
fi fi
} }
function install_taos-explorer_service() {
if ((${service_mod}==0)); then
[ -f ${script_dir}/../cfg/taos-explorer.service ] &&\
${csudo}cp ${script_dir}/../cfg/taos-explorer.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
}
function install_service() { function install_service() {
log_print "start install service" log_print "start install service"
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
@ -748,8 +771,10 @@ function install_TDengine() {
install_config install_config
install_taosadapter_config install_taosadapter_config
install_taoskeeper_config install_taoskeeper_config
install_taos-explorer_config
install_taosadapter_service install_taosadapter_service
install_taoskeeper_service install_taoskeeper_service
install_taos-explorer_service
install_service install_service
install_app install_app
@ -757,40 +782,38 @@ function install_TDengine() {
#echo #echo
#echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" #echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo echo
echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" echo -e "${GREEN_DARK}To configure TDengine ${NC}\t: edit /etc/taos/taos.cfg"
echo -e "${GREEN_DARK}To configure taosAdapter ${NC}\t: edit /etc/taos/taosadapter.toml"
echo -e "${GREEN_DARK}To configure taos-explorer ${NC}\t: edit /etc/taos/explorer.toml"
if ((${service_mod}==0)); then if ((${service_mod}==0)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}systemctl start taosd${NC}" echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}systemctl start taosd${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}systemctl start taosadapter${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}systemctl start taoskeeper${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}systemctl start taos-explorer${NC}"
elif ((${service_mod}==1)); then elif ((${service_mod}==1)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}" echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd start ${RED} after${NC}" echo -e " : ${csudo}service taosd start ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}update-rc.d taosadapter default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taosadapter ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}update-rc.d taoskeeper default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taoskeeper ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}update-rc.d taos-explorer default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taos-explorer ${RED} after${NC}"
else else
echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}" echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ./taosd${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ./taosadapter${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ./taoskeeper${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ./taos-explorer${NC}"
fi fi
if [ ! -z "$firstEp" ]; then
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
echo
fi
log_print "install TDengine successfully!" log_print "install TDengine successfully!"
echo echo
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo "TDengine is installed successfully!"
echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : taos -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
} }

359
packaging/tools/remove.sh
View File

@ -31,25 +31,22 @@ else
lib64_link_dir="/usr/local/lib" lib64_link_dir="/usr/local/lib"
inc_link_dir="/usr/local/include" inc_link_dir="/usr/local/include"
fi fi
serverName="taosd"
clientName="taos" PREFIX="taos"
uninstallScript="rmtaos" serverName="${PREFIX}d"
clientName="${PREFIX}"
uninstallScript="rm${PREFIX}"
adapterName="${PREFIX}adapter"
demoName="${PREFIX}demo"
benchmarkName="${PREFIX}Benchmark"
dumpName="${PREFIX}dump"
keeperName="${PREFIX}keeper"
xName="${PREFIX}x"
explorerName="${PREFIX}-explorer"
tarbitratorName="tarbitratord"
productName="TDengine" productName="TDengine"
serverName2="taosd" installDir="/usr/local/${PREFIX}"
clientName2="taos"
productName2="TDengine"
adapterName2="${clientName2}adapter"
demoName2="${clientName2}demo"
benchmarkName2="${clientName2}Benchmark"
dumpName2="${clientName2}dump"
keeperName2="${clientName2}keeper"
xName2="${clientName2}x"
explorerName2="${clientName2}-explorer"
uninstallScript2="rm${clientName2}"
installDir="/usr/local/${clientName2}"
#install main path #install main path
install_main_dir=${installDir} install_main_dir=${installDir}
@ -57,15 +54,11 @@ data_link_dir=${installDir}/data
log_link_dir=${installDir}/log log_link_dir=${installDir}/log
cfg_link_dir=${installDir}/cfg cfg_link_dir=${installDir}/cfg
local_bin_link_dir="/usr/local/bin" local_bin_link_dir="/usr/local/bin"
service_config_dir="/etc/systemd/system" service_config_dir="/etc/systemd/system"
taos_service_name=${serverName2} config_dir="/etc/${PREFIX}"
taosadapter_service_name="${clientName2}adapter"
tarbitrator_service_name="tarbitratord"
config_dir="/etc/${clientName2}"
services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"x" ${PREFIX}"-explorer" ${PREFIX}"keeper")
tools=(${PREFIX} ${PREFIX}"Benchmark" ${PREFIX}"dump" ${PREFIX}"demo" udfd set_core.sh TDinsight.sh $uninstallScript start-all.sh stop-all.sh)
csudo="" csudo=""
if command -v sudo >/dev/null; then if command -v sudo >/dev/null; then
@ -92,215 +85,26 @@ else
service_mod=2 service_mod=2
fi fi
function kill_taosadapter() { kill_service_of() {
pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_taosd() {
pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_tarbitrator() {
pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function clean_bin() {
# Remove link
${csudo}rm -f ${bin_link_dir}/${clientName} || :
${csudo}rm -f ${bin_link_dir}/${serverName} || :
echo "${serverName} is removed successfully"
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/${adapterName2} || :
echo "${adapterName2} is removed successfully"
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
${csudo}rm -f ${bin_link_dir}/${keeperName2} || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${xName2} || :
${csudo}rm -f ${bin_link_dir}/${explorerName2} || :
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
fi
}
function clean_local_bin() {
${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${local_bin_link_dir}/${demoName2} || :
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
# Remove link
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {
# Remove link
${csudo}rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo}rm -rf ${log_link_dir} || :
}
function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
if systemctl is-active --quiet ${taos_service_name}; then
echo "${taos_service_name} is running, stopping it..."
${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${taosd_service_config}
taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service"
if systemctl is-active --quiet ${taosadapter_service_name}; then
echo "${clientName2}Adapter is running, stopping it..."
${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
[ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
if systemctl is-active --quiet ${tarbitrator_service_name}; then
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
}
function clean_service_on_sysvinit() {
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
echo "${serverName2} is running, stopping it..."
${csudo}service ${serverName} stop || :
fi
if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}service tarbitratord stop || :
fi
if ((${initd_mod} == 1)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}chkconfig --del ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}chkconfig --del tarbitratord || :
fi
elif ((${initd_mod} == 2)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}insserv -r ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}insserv -r tarbitratord || :
fi
elif ((${initd_mod} == 3)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}update-rc.d -f ${serverName} remove || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}update-rc.d -f tarbitratord remove || :
fi
fi
${csudo}rm -f ${service_config_dir}/${serverName} || :
${csudo}rm -f ${service_config_dir}/tarbitratord || :
if $(which init &>/dev/null); then
${csudo}init q || :
fi
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
}
function clean_service() {
if ((${service_mod} == 0)); then
clean_service_on_systemd
elif ((${service_mod} == 1)); then
clean_service_on_sysvinit
else
if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
fi
kill_taosadapter
kill_taosd
kill_tarbitrator
fi
}
function remove_data_and_config() {
data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$data_dir" == X"" ]; then
data_dir="/var/lib/taos"
fi
log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$log_dir" == X"" ]; then
log_dir="/var/log/taos"
fi
[ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}/*
[ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}/*
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}/*
}
_kill_service_of() {
_service=$1 _service=$1
pid=$(ps -ef | grep "$_service" | grep -v "grep" | awk '{print $2}') pid=$(ps -ef | grep $_service | grep -v grep | grep -v $uninstallScript | awk '{print $2}')
if [ -n "$pid" ]; then if [ -n "$pid" ]; then
${csudo}kill -9 $pid || : ${csudo}kill -9 $pid || :
fi fi
} }
_clean_service_on_systemd_of() { clean_service_on_systemd_of() {
_service=$1 _service=$1
_service_config="${service_config_dir}/${_service}.service" _service_config="${service_config_dir}/${_service}.service"
if systemctl is-active --quiet ${_service}; then if systemctl is-active --quiet ${_service}; then
echo "taoskeeper is running, stopping it..." echo "${_service} is running, stopping it..."
${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null ${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null
fi fi
${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null ${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${_service_config} ${csudo}rm -f ${_service_config}
} }
_clean_service_on_sysvinit_of() {
clean_service_on_sysvinit_of() {
_service=$1 _service=$1
if pidof ${_service} &>/dev/null; then if pidof ${_service} &>/dev/null; then
echo "${_service} is running, stopping it..." echo "${_service} is running, stopping it..."
@ -327,45 +131,99 @@ _clean_service_on_sysvinit_of() {
fi fi
} }
_clean_service_of() { clean_service_of() {
_service=$1 _service=$1
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
_clean_service_on_systemd_of $_service clean_service_on_systemd_of $_service
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
_clean_service_on_sysvinit_of $_service clean_service_on_sysvinit_of $_service
else else
_kill_service_of $_service kill_service_of $_service
fi fi
} }
remove_taoskeeper() { remove_service_of() {
# remove taoskeeper bin _service=$1
_clean_service_of taoskeeper clean_service_of ${_service}
[ -e "${bin_link_dir}/taoskeeper" ] && ${csudo}rm -rf ${bin_link_dir}/taoskeeper if [[ -e "${bin_link_dir}/${_service}" || -e "${installDir}/bin/${_service}" || -e "${local_bin_link_dir}/${_service}" ]]; then
[ -e "${installDir}/taoskeeper" ] && ${csudo}rm -rf ${installDir}/taoskeeper ${csudo}rm -rf ${bin_link_dir}/${_service}
[ -e "${cfg_link_dir}/metrics.toml" ] || ${csudo}rm -rf ${cfg_link_dir}/metrics.toml ${csudo}rm -rf ${installDir}/bin/${_service}
echo "taosKeeper is removed successfully!" ${csudo}rm -rf ${local_bin_link_dir}/${_service}
} echo "${_service} is removed successfully!"
function uninstall_taosx() {
if [ -f ${installDir}/uninstall.sh ]; then
cd ${installDir}
bash uninstall.sh
fi fi
} }
if [ "$verMode" == "cluster" ]; then remove_tools_of() {
uninstall_taosx _tool=$1
fi kill_service_of ${_tool}
[ -e "${bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${bin_link_dir}/${_tool} || :
[ -e "${installDir}/bin/${_tool}" ] && ${csudo}rm -rf ${installDir}/bin/${_tool} || :
[ -e "${local_bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${local_bin_link_dir}/${_tool} || :
}
remove_taoskeeper remove_bin() {
# Stop service and disable booting start. for _service in "${services[@]}"; do
clean_service remove_service_of ${_service}
# Remove binary file and links done
clean_bin
# Remove links of local bin for _tool in "${tools[@]}"; do
clean_local_bin remove_tools_of ${_tool}
# Remove header file. done
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
# Remove link
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {
# Remove link
${csudo}rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo}rm -rf ${log_link_dir} || :
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
}
function remove_data_and_config() {
data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$data_dir" == X"" ]; then
data_dir="/var/lib/${PREFIX}"
fi
log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$log_dir" == X"" ]; then
log_dir="/var/log/${PREFIX}"
fi
[ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}
[ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}
}
remove_bin
clean_header clean_header
# Remove lib file # Remove lib file
clean_lib clean_lib
@ -375,8 +233,7 @@ clean_log
clean_config clean_config
# Remove data link directory # Remove data link directory
${csudo}rm -rf ${data_link_dir} || : ${csudo}rm -rf ${data_link_dir} || :
${csudo}rm -rf ${install_main_dir} || :
${csudo}rm -rf ${install_main_dir}
if [[ -e /etc/os-release ]]; then if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
else else
@ -394,6 +251,7 @@ elif echo $osinfo | grep -qwi "centos"; then
${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || :
fi fi
if [ "$osType" = "Darwin" ]; then if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
${csudo}rm -rf /Applications/TDengine.app ${csudo}rm -rf /Applications/TDengine.app
fi fi
@ -411,6 +269,7 @@ if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then
fi fi
fi fi
command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true
echo echo
echo "${productName2} is removed successfully!" echo "${productName} is removed successfully!"
echo echo

3
packaging/tools/remove_client.sh
View File

@ -38,7 +38,8 @@ if command -v sudo > /dev/null; then
fi fi
function kill_client() { function kill_client() {
if [ -n "$(ps aux | grep -v grep | grep ${clientName})" ]; then pid=$(ps -ef | grep ${clientName2} | grep -v grep | grep -v $uninstallScript2 | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || : ${csudo}kill -9 $pid || :
fi fi
} }

6
source/client/src/clientEnv.c
View File

@ -885,7 +885,11 @@ void taos_init_imp(void) {
taosSetCoreDump(true); taosSetCoreDump(true);
#endif #endif
initTaskQueue(); if (initTaskQueue() != 0){
tscInitRes = -1;
tscError("failed to init task queue");
return;
}
fmFuncMgtInit(); fmFuncMgtInit();
nodesInitAllocatorSet(); nodesInitAllocatorSet();

2
source/client/src/clientHb.c
View File

@ -744,7 +744,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
} }
if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) != 0) {
taosMemoryFree(user); taosMemoryFree(user);
code = terrno ? terrno : TSDB_CODE_APP_ERROR; code = terrno ? terrno : TSDB_CODE_APP_ERROR;
goto _return; goto _return;

3
source/client/src/clientImpl.c
View File

@ -2007,7 +2007,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst))); varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (tTagIsJson(data)) { } else if (tTagIsJson(data)) {
char* jsonString = parseTagDatatoJson(data); char* jsonString = NULL;
parseTagDatatoJson(data, &jsonString);
STR_TO_VARSTR(dst, jsonString); STR_TO_VARSTR(dst, jsonString);
taosMemoryFree(jsonString); taosMemoryFree(jsonString);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"

2
source/client/src/clientMonitor.c
View File

@ -691,7 +691,7 @@ static void* monitorThreadFunc(void* param) {
tscDebug("monitorThreadFunc start"); tscDebug("monitorThreadFunc start");
int64_t quitTime = 0; int64_t quitTime = 0;
while (1) { while (1) {
if (atomic_load_32(&slowLogFlag) > 0 > 0) { if (atomic_load_32(&slowLogFlag) > 0) {
if (quitCnt == 0) { if (quitCnt == 0) {
monitorSendAllSlowLogAtQuit(); monitorSendAllSlowLogAtQuit();
if (quitCnt == 0) { if (quitCnt == 0) {

1220
source/client/src/clientRawBlockWrite.c
View File

File diff suppressed because it is too large Load Diff

20
source/client/test/CMakeLists.txt
View File

@ -23,11 +23,11 @@ TARGET_LINK_LIBRARIES(
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
) )
ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp) #ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
TARGET_LINK_LIBRARIES( #TARGET_LINK_LIBRARIES(
clientMonitorTest # clientMonitorTest
PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor # PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
) #)
TARGET_INCLUDE_DIRECTORIES( TARGET_INCLUDE_DIRECTORIES(
clientTest clientTest
@ -47,11 +47,11 @@ TARGET_INCLUDE_DIRECTORIES(
PRIVATE "${TD_SOURCE_DIR}/source/client/inc" PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
) )
TARGET_INCLUDE_DIRECTORIES( #TARGET_INCLUDE_DIRECTORIES(
clientMonitorTest # clientMonitorTest
PUBLIC "${TD_SOURCE_DIR}/include/client/" # PUBLIC "${TD_SOURCE_DIR}/include/client/"
PRIVATE "${TD_SOURCE_DIR}/source/client/inc" # PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
) #)
add_test( add_test(
NAME smlTest NAME smlTest

26
source/common/src/rsync.c
View File

@ -54,10 +54,11 @@ static void changeDirFromWindowsToLinux(char* from, char* to) {
#endif #endif
static int32_t generateConfigFile(char* confDir) { static int32_t generateConfigFile(char* confDir) {
int32_t code = 0;
TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -92,7 +93,8 @@ static int32_t generateConfigFile(char* confDir) {
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) { if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; code = TAOS_SYSTEM_ERROR(errno);
return code;
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
@ -129,11 +131,13 @@ void stopRsync() {
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation. taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
} }
void startRsync() { int32_t startRsync() {
int32_t code = 0;
if (taosMulMkDir(tsCheckpointBackupDir) != 0) { if (taosMulMkDir(tsCheckpointBackupDir) != 0) {
uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir, uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir,
ERRNO_ERR_DATA); ERRNO_ERR_DATA);
return; code = TAOS_SYSTEM_ERROR(errno);
return code;
} }
removeEmptyDir(); removeEmptyDir();
@ -141,9 +145,9 @@ void startRsync() {
char confDir[PATH_MAX] = {0}; char confDir[PATH_MAX] = {0};
snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir); snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
int32_t code = generateConfigFile(confDir); code = generateConfigFile(confDir);
if (code != 0) { if (code != 0) {
return; return code;
} }
char cmd[PATH_MAX] = {0}; char cmd[PATH_MAX] = {0};
@ -152,9 +156,11 @@ void startRsync() {
code = system(cmd); code = system(cmd);
if (code != 0) { if (code != 0) {
uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA); uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
uInfo("[rsync] cmd:%s start server successful", cmd); uInfo("[rsync] cmd:%s start server successful", cmd);
} }
return code;
} }
int32_t uploadByRsync(const char* id, const char* path) { int32_t uploadByRsync(const char* id, const char* path) {
@ -199,6 +205,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) { if (code != 0) {
uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA); tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
int64_t el = (taosGetTimestampMs() - st); int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -242,6 +249,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) { if (code != 0) {
uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA); tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
int64_t el = (taosGetTimestampMs() - st); int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -283,13 +291,13 @@ int32_t downloadRsync(const char* id, const char* path) {
uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id, uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
path, times, code, ERRNO_ERR_DATA); path, times, code, ERRNO_ERR_DATA);
taosSsleep(1); taosSsleep(1);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
int32_t el = taosGetTimestampMs() - st; int32_t el = taosGetTimestampMs() - st;
uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el); uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
break; break;
} }
} }
return code; return code;
} }
@ -298,7 +306,7 @@ int32_t deleteRsync(const char* id) {
int32_t code = taosMkDir(tmp); int32_t code = taosMkDir(tmp);
if (code != 0) { if (code != 0) {
uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
char command[PATH_MAX] = {0}; char command[PATH_MAX] = {0};
@ -310,7 +318,7 @@ int32_t deleteRsync(const char* id) {
taosRemoveDir(tmp); taosRemoveDir(tmp);
if (code != 0) { if (code != 0) {
uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
uDebug("[rsync] delete data:%s successful", id); uDebug("[rsync] delete data:%s successful", id);

12
source/common/src/tdatablock.c
View File

@ -1523,9 +1523,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
} }
void* blockDataDestroy(SSDataBlock* pBlock) { void blockDataDestroy(SSDataBlock* pBlock) {
if (pBlock == NULL) { if (pBlock == NULL) {
return NULL; return;
} }
if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) { if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) {
@ -1535,7 +1535,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
blockDataFreeRes(pBlock); blockDataFreeRes(pBlock);
taosMemoryFreeClear(pBlock); taosMemoryFreeClear(pBlock);
return NULL;
} }
// todo remove it // todo remove it
@ -2468,19 +2467,18 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) { int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) {
if (stbFullName[0] == 0) { if (stbFullName[0] == 0) {
terrno = TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
} }
SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); SArray* tags = taosArrayInit(0, sizeof(SSmlKv));
if (tags == NULL) { if (tags == NULL) {
return TSDB_CODE_FAILED; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (cname == NULL) { if (cname == NULL) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
taosArrayDestroy(tags); taosArrayDestroy(tags);
return TSDB_CODE_FAILED; return terrno;
} }
int8_t type = TSDB_DATA_TYPE_UBIGINT; int8_t type = TSDB_DATA_TYPE_UBIGINT;

58
source/common/src/tglobal.c
View File

@ -1370,58 +1370,68 @@ static int32_t taosCheckGlobalCfg() {
return 0; return 0;
} }
static int32_t cfgInitWrapper(SConfig **pCfg) {
if (*pCfg == NULL) {
*pCfg = cfgInit();
if (*pCfg == NULL) {
return terrno;
}
}
return 0;
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) { bool tsc) {
if (tsCfg != NULL) return 0; if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
int32_t code = cfgInitWrapper(&tsCfg);
if (tsc) { if (tsc) {
if (taosAddClientCfg(tsCfg) != 0) return -1; if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if (taosAddClientLogCfg(tsCfg) != 0) return -1; if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
} else { } else {
if (taosAddClientCfg(tsCfg) != 0) return -1; if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if (taosAddServerCfg(tsCfg) != 0) return -1; if ((code = taosAddServerCfg(tsCfg)) != 0) return code;
if (taosAddClientLogCfg(tsCfg) != 0) return -1; if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
if (taosAddServerLogCfg(tsCfg) != 0) return -1; if ((code = taosAddServerLogCfg(tsCfg)) != 0) return code;
} }
taosAddSystemCfg(tsCfg); code = taosAddSystemCfg(tsCfg);
if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
uError("failed to load cfg since %s", terrstr()); uError("failed to load cfg since %s", tstrerror(code));
cfgCleanup(tsCfg); cfgCleanup(tsCfg);
tsCfg = NULL; tsCfg = NULL;
return -1; return code;
} }
if (cfgLoadFromArray(tsCfg, pArgs) != 0) { if ((code = cfgLoadFromArray(tsCfg, pArgs)) != 0) {
uError("failed to load cfg from array since %s", terrstr()); uError("failed to load cfg from array since %s", tstrerror(code));
cfgCleanup(tsCfg); cfgCleanup(tsCfg);
tsCfg = NULL; tsCfg = NULL;
return -1; return code;
} }
if (tsc) { if (tsc) {
if (taosSetClientCfg(tsCfg)) return -1; if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
} else { } else {
if (taosSetClientCfg(tsCfg)) return -1; if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
if (taosUpdateServerCfg(tsCfg)) return -1; if ((code = taosUpdateServerCfg(tsCfg)) != 0) return code;
if (taosSetServerCfg(tsCfg)) return -1; if ((code = taosSetServerCfg(tsCfg)) != 0) return code;
if (taosSetReleaseCfg(tsCfg)) return -1; if ((code = taosSetReleaseCfg(tsCfg)) != 0) return code;
if (taosSetTfsCfg(tsCfg) != 0) return -1; if ((code = taosSetTfsCfg(tsCfg)) != 0) return code;
if (taosSetS3Cfg(tsCfg) != 0) return -1; if ((code = taosSetS3Cfg(tsCfg)) != 0) return code;
} }
taosSetSystemCfg(tsCfg); taosSetSystemCfg(tsCfg);
if (taosSetFileHandlesLimit() != 0) return -1; if ((code = taosSetFileHandlesLimit()) != 0) return code;
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
cfgDumpCfg(tsCfg, tsc, false); cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) { if ((code = taosCheckGlobalCfg()) != 0) {
return -1; return code;
} }
return 0; return 0;

12
source/common/src/tmsg.c
View File

@ -9258,6 +9258,12 @@ void tOffsetDestroy(void *param) {
taosMemoryFreeClear(pVal->primaryKey.pData); taosMemoryFreeClear(pVal->primaryKey.pData);
} }
} }
void tDeleteSTqOffset(void *param) {
STqOffset *pVal = (STqOffset *)param;
tOffsetDestroy(&pVal->val);
}
int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1;
if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1;
@ -9297,12 +9303,12 @@ int32_t tEncodeSTqCheckInfo(SEncoder *pEncoder, const STqCheckInfo *pInfo) {
int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) { int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) {
if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1; if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1;
if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1; if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1;
int32_t sz; int32_t sz = 0;
if (tDecodeI32(pDecoder, &sz) < 0) return -1; if (tDecodeI32(pDecoder, &sz) < 0) return -1;
pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t)); pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t));
if (pInfo->colIdList == NULL) return -1; if (pInfo->colIdList == NULL) return -1;
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
int16_t colId; int16_t colId = 0;
if (tDecodeI16(pDecoder, &colId) < 0) return -1; if (tDecodeI16(pDecoder, &colId) < 0) return -1;
taosArrayPush(pInfo->colIdList, &colId); taosArrayPush(pInfo->colIdList, &colId);
} }
@ -10822,7 +10828,9 @@ int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
if (tDecodeI32(pDecoder, &size) < 0) return -1; if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) { if (size > 0) {
pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES); pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES);
if (!pRsp->batchMetaReq) return -1;
pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t)); pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t));
if (!pRsp->batchMetaLen) return -1;
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
void *pCreate = NULL; void *pCreate = NULL;
uint64_t len = 0; uint64_t len = 0;

77
source/dnode/mgmt/exe/dmMain.c
View File

@ -167,23 +167,23 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow"); printf("config file path overflow");
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(configDir, argv[i], PATH_MAX); tstrncpy(configDir, argv[i], PATH_MAX);
} else { } else {
printf("'-c' requires a parameter, default is %s\n", configDir); printf("'-c' requires a parameter, default is %s\n", configDir);
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow"); printf("apollo url overflow");
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(global.apolloUrl, argv[i], PATH_MAX); tstrncpy(global.apolloUrl, argv[i], PATH_MAX);
} else { } else {
printf("'-a' requires a parameter\n"); printf("'-a' requires a parameter\n");
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-s") == 0) { } else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true; global.dumpSdb = true;
@ -191,31 +191,31 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow"); printf("env file path overflow");
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(global.envFile, argv[i], PATH_MAX); tstrncpy(global.envFile, argv[i], PATH_MAX);
} else { } else {
printf("'-E' requires a parameter\n"); printf("'-E' requires a parameter\n");
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-k") == 0) { } else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true; global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) { } else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true; global.generateCode = true;
if(i < argc - 1) { if (i < argc - 1) {
int32_t len = strlen(argv[++i]); int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) { if (len < ENCRYPT_KEY_LEN_MIN) {
printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN); printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1; return TSDB_CODE_INVALID_CFG;
} }
if (len > ENCRYPT_KEY_LEN) { if (len > ENCRYPT_KEY_LEN) {
printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN); printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN); tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else { } else {
printf("'-y' requires a parameter\n"); printf("'-y' requires a parameter\n");
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-C") == 0) { } else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true; global.dumpConfig = true;
@ -310,6 +310,7 @@ static void taosCleanupArgs() {
} }
int main(int argc, char const *argv[]) { int main(int argc, char const *argv[]) {
int32_t code = 0;
#ifdef TD_JEMALLOC_ENABLED #ifdef TD_JEMALLOC_ENABLED
bool jeBackgroundThread = true; bool jeBackgroundThread = true;
mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool)); mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool));
@ -319,10 +320,10 @@ int main(int argc, char const *argv[]) {
return -1; return -1;
} }
if (dmParseArgs(argc, argv) != 0) { if ((code = dmParseArgs(argc, argv)) != 0) {
//printf("failed to start since parse args error\n"); // printf("failed to start since parse args error\n");
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -335,6 +336,7 @@ int main(int argc, char const *argv[]) {
return 0; return 0;
} }
int mainWindows(int argc, char **argv) { int mainWindows(int argc, char **argv) {
int32_t code = 0;
#endif #endif
if (global.generateGrant) { if (global.generateGrant) {
@ -357,7 +359,7 @@ int mainWindows(int argc, char **argv) {
#if defined(LINUX) #if defined(LINUX)
if (global.memDbg) { if (global.memDbg) {
int32_t code = taosMemoryDbgInit(); code = taosMemoryDbgInit();
if (code) { if (code) {
printf("failed to init memory dbg, error:%s\n", tstrerror(code)); printf("failed to init memory dbg, error:%s\n", tstrerror(code));
return code; return code;
@ -366,16 +368,18 @@ int mainWindows(int argc, char **argv) {
printf("memory dbg enabled\n"); printf("memory dbg enabled\n");
} }
#endif #endif
if(global.generateCode) { if (global.generateCode) {
bool toLogFile = false; bool toLogFile = false;
if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){ if ((code = taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs)) != 0) {
encryptError("failed to generate encrypt code since dataDir can not be set from cfg file"); encryptError("failed to generate encrypt code since dataDir can not be set from cfg file,reason:%s",
return -1; tstrerror(code));
return code;
}; };
TdFilePtr pFile;
if(dmCheckRunning(tsDataDir) == NULL) { if ((code = dmCheckRunning(tsDataDir, &pFile)) != 0) {
encryptError("failed to generate encrypt code since taosd is running, please stop it first"); encryptError("failed to generate encrypt code since taosd is running, please stop it first, reason:%s",
return -1; tstrerror(code));
return code;
} }
int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile); int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile);
taosCloseLog(); taosCloseLog();
@ -383,30 +387,30 @@ int mainWindows(int argc, char **argv) {
return ret; return ret;
} }
if (dmInitLog() != 0) { if ((code = dmInitLog()) != 0) {
printf("failed to start since init log error\n"); printf("failed to start since init log error\n");
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
dmPrintArgs(argc, argv); dmPrintArgs(argc, argv);
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { if ((code = taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0)) != 0) {
dError("failed to start since read config error"); dError("failed to start since read config error");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
if (taosConvInit() != 0) { if ((code = taosConvInit()) != 0) {
dError("failed to init conv"); dError("failed to init conv");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
if (global.checkS3) { if (global.checkS3) {
int32_t code = dmCheckS3(); code = dmCheckS3();
taosCleanupCfg(); taosCleanupCfg();
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
@ -435,31 +439,32 @@ int mainWindows(int argc, char **argv) {
osSetProcPath(argc, (char **)argv); osSetProcPath(argc, (char **)argv);
taosCleanupArgs(); taosCleanupArgs();
if(dmGetEncryptKey() != 0){ if ((code = dmGetEncryptKey()) != 0) {
dError("failed to start since failed to get encrypt key"); dError("failed to start since failed to get encrypt key");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
}; };
if (dmInit() != 0) { if ((code = dmInit()) != 0) {
if (terrno == TSDB_CODE_NOT_FOUND) { if (code == TSDB_CODE_NOT_FOUND) {
dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support"); dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support");
} else { } else {
dError("failed to init dnode since %s", terrstr()); dError("failed to init dnode since %s", tstrerror(code));
} }
taosCleanupCfg(); taosCleanupCfg();
taosCloseLog(); taosCloseLog();
taosConvDestroy(); taosConvDestroy();
return -1; return code;
} }
dInfo("start to init service"); dInfo("start to init service");
dmSetSignalHandle(); dmSetSignalHandle();
tsDndStart = taosGetTimestampMs(); tsDndStart = taosGetTimestampMs();
tsDndStartOsUptime = taosGetOsUptime(); tsDndStartOsUptime = taosGetOsUptime();
int32_t code = dmRun();
code = dmRun();
dInfo("shutting down the service"); dInfo("shutting down the service");
dmCleanup(); dmCleanup();

115
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -45,7 +45,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer}; SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer};
int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req); int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen); void *pHead = rpcMallocCont(contLen);
tSerializeRetrieveIpWhite(pHead, contLen, &req); tSerializeRetrieveIpWhite(pHead, contLen, &req);
SRpcMsg rpcMsg = {.pCont = pHead, SRpcMsg rpcMsg = {.pCont = pHead,
@ -146,7 +146,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.ipWhiteVer = pMgmt->pData->ipWhiteVer; req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen); void *pHead = rpcMallocCont(contLen);
tSerializeSStatusReq(pHead, contLen, &req); tSerializeSStatusReq(pHead, contLen, &req);
tFreeSStatusReq(&req); tFreeSStatusReq(&req);
@ -207,18 +207,26 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0}; SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
SConfig *pCfg = taosGetCfg(); SConfig *pCfg = taosGetCfg();
cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
taosCfgDynamicOptions(pCfg, cfgReq.config, true); code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
return 0; if (code != 0) {
if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) {
code = 0;
} else {
return code;
}
}
return taosCfgDynamicOptions(pCfg, cfgReq.config, true);
} }
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@ -276,32 +284,49 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
} }
int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
dDebug("server run status req is received"); dDebug("server run status req is received");
SServerStatusRsp statusRsp = {0}; SServerStatusRsp statusRsp = {0};
dmGetServerRunStatus(pMgmt, &statusRsp); dmGetServerRunStatus(pMgmt, &statusRsp);
pMsg->info.rsp = NULL;
pMsg->info.rspLen = 0;
SRpcMsg rspMsg = {.info = pMsg->info}; SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) { if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1; // rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
} }
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1; // rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
}
rspLen = tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
if (rspLen < 0) {
return TSDB_CODE_INVALID_MSG;
} }
tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
pMsg->info.rsp = pRsp; pMsg->info.rsp = pRsp;
pMsg->info.rspLen = rspLen; pMsg->info.rspLen = rspLen;
return 0; return 0;
} }
SSDataBlock *dmBuildVariablesBlock(void) { int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) {
SSDataBlock * pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); int32_t code = 0;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
size_t size = 0; size_t size = 0;
const SSysTableMeta *pMeta = NULL; const SSysTableMeta *pMeta = NULL;
getInfosDbMeta(&pMeta, &size); getInfosDbMeta(&pMeta, &size);
@ -314,52 +339,74 @@ SSDataBlock *dmBuildVariablesBlock(void) {
} }
pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData)); pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
for (int32_t i = 0; i < pMeta[index].colNum; ++i) { for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData = {0}; SColumnInfoData colInfoData = {0};
colInfoData.info.colId = i + 1; colInfoData.info.colId = i + 1;
colInfoData.info.type = pMeta[index].schema[i].type; colInfoData.info.type = pMeta[index].schema[i].type;
colInfoData.info.bytes = pMeta[index].schema[i].bytes; colInfoData.info.bytes = pMeta[index].schema[i].bytes;
taosArrayPush(pBlock->pDataBlock, &colInfoData); if (taosArrayPush(pBlock->pDataBlock, &colInfoData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
pBlock->info.hasVarCol = true; pBlock->info.hasVarCol = true;
_exit:
return pBlock; if (code != 0) {
blockDataDestroy(pBlock);
} else {
*ppBlock = pBlock;
}
return code;
} }
int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
/*int32_t code = */dumpConfToDataBlock(pBlock, 1); int32_t code = dumpConfToDataBlock(pBlock, 1);
if (code != 0) {
return code;
}
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false); if (pColInfo == NULL) {
return TSDB_CODE_OUT_OF_RANGE;
}
return TSDB_CODE_SUCCESS; return colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
} }
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t size = 0; int32_t size = 0;
int32_t rowsRead = 0; int32_t rowsRead = 0;
int32_t code = 0;
SRetrieveTableReq retrieveReq = {0}; SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) { if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
#if 0 #if 0
if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
terrno = TSDB_CODE_MND_NO_RIGHTS; code = TSDB_CODE_MND_NO_RIGHTS;
return -1; return code;
} }
#endif #endif
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
SSDataBlock *pBlock = dmBuildVariablesBlock(); SSDataBlock *pBlock = NULL;
if ((code = dmBuildVariablesBlock(&pBlock)) != 0) {
return code;
}
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); code = dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
if (code != 0) {
blockDataDestroy(pBlock);
return code;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
@ -367,10 +414,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to retrieve data since %s", terrstr()); dError("failed to retrieve data since %s", tstrerror(code));
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; return code;
} }
char *pStart = pRsp->data; char *pStart = pRsp->data;
@ -404,7 +451,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SArray *dmGetMsgHandles() { SArray *dmGetMsgHandles() {
int32_t code = -1; int32_t code = -1;
SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle)); SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));
if (pArray == NULL) goto _OVER; if (pArray == NULL) {
return NULL;
}
// Requests handled by DNODE // Requests handled by DNODE
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;

32
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -18,22 +18,23 @@
#include "libs/function/tudf.h" #include "libs/function/tudf.h"
static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartStatusThread(pMgmt) != 0) { int32_t code = 0;
return -1; if ((code = dmStartStatusThread(pMgmt)) != 0) {
return code;
} }
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (dmStartNotifyThread(pMgmt) != 0) { if ((code = dmStartNotifyThread(pMgmt)) != 0) {
return -1; return code;
} }
#endif #endif
if (dmStartMonitorThread(pMgmt) != 0) { if ((code = dmStartMonitorThread(pMgmt)) != 0) {
return -1; return code;
} }
if (dmStartAuditThread(pMgmt) != 0) { if ((code = dmStartAuditThread(pMgmt)) != 0) {
return -1; return code;
} }
if (dmStartCrashReportThread(pMgmt) != 0) { if ((code = dmStartCrashReportThread(pMgmt)) != 0) {
return -1; return code;
} }
return 0; return 0;
} }
@ -50,10 +51,10 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) {
} }
static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -70,12 +71,11 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp; pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp;
// pMgmt->pData->ipWhiteVer = 0; if ((code = dmStartWorker(pMgmt)) != 0) {
if (dmStartWorker(pMgmt) != 0) { return code;
return -1;
} }
if (udfStartUdfd(pMgmt->pData->dnodeId) != 0) { if ((code = udfStartUdfd(pMgmt->pData->dnodeId)) != 0) {
dError("failed to start udfd"); dError("failed to start udfd");
} }

34
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -264,12 +264,14 @@ static void *dmCrashReportThreadFp(void *param) {
} }
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) {
dError("failed to create status thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create status thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -285,12 +287,14 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) {
dError("failed to create notify thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create notify thread since %s", strerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -308,12 +312,14 @@ void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) {
dError("failed to create monitor thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create monitor thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -322,12 +328,14 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) { int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
dError("failed to create audit thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create audit thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -350,6 +358,7 @@ void dmStopAuditThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) { int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
if (!tsEnableCrashReport) { if (!tsEnableCrashReport) {
return 0; return 0;
} }
@ -358,8 +367,9 @@ int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) {
dError("failed to create crashReport thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create crashReport thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -431,8 +441,8 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg); code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg);
break; break;
default: default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg); dGError("msg:%p, not processed in mgmt queue, reason:%s", pMsg, tstrerror(code));
break; break;
} }

135
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -21,11 +21,11 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
int32_t code = 0; int32_t code = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code); tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code);
if (code < 0) return -1; if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code);
if (code < 0) return 0; if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code); tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code);
if (code < 0) return 0; if (code < 0) return code;
SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); SJson *replicas = tjsonGetObjectItem(pJson, "replicas");
if (replicas == NULL) return 0; if (replicas == NULL) return 0;
@ -35,17 +35,17 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonGetArrayItem(replicas, i); SJson *replica = tjsonGetArrayItem(replicas, i);
if (replica == NULL) return -1; if (replica == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
SReplica *pReplica = pOption->replicas + i; SReplica *pReplica = pOption->replicas + i;
tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code); tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code);
if (code < 0) return -1; if (code < 0) return code;
code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn); code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn);
if (code < 0) return -1; if (code < 0) return code;
tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code);
if (code < 0) return -1; if (code < 0) return code;
tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code); tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
if (code < 0) return -1; if (code < 0) return code;
if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) { if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
pOption->numOfReplicas++; pOption->numOfReplicas++;
} }
@ -63,36 +63,41 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char *pData = NULL; char *pData = NULL;
SJson *pJson = NULL; SJson *pJson = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file); dInfo("mnode file:%s not exist, reason:%s", file, tstrerror(TAOS_SYSTEM_ERROR(errno)));
return 0; return 0;
} }
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open mnode file:%s since %s", file, terrstr()); dError("failed to open mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr()); dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
pData = taosMemoryMalloc(size + 1); pData = taosMemoryMalloc(size + 1);
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, pData, size) != size) { if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read mnode file:%s since %s", file, terrstr()); dError("failed to read mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -100,12 +105,11 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
pJson = tjsonParse(pData); pJson = tjsonParse(pData);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (mmDecodeOption(pJson, pOption) < 0) { if ((code = mmDecodeOption(pJson, pOption)) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -118,37 +122,42 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read mnode file:%s since %s", file, terrstr()); dError("failed to read mnode file:%s since %s", file, tstrerror(code));
} }
return code; return code;
} }
static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) {
int32_t code = 0;
if (pOption->deploy && pOption->numOfTotalReplicas > 0) { if (pOption->deploy && pOption->numOfTotalReplicas > 0) {
if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1; if ((code = tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex)) < 0) return code;
SJson *replicas = tjsonCreateArray(); SJson *replicas = tjsonCreateArray();
if (replicas == NULL) return -1; if (replicas == NULL) {
if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1; return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "replicas", replicas)) < 0) return code;
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonCreateObject(); SJson *replica = tjsonCreateObject();
if (replica == NULL) return -1; if (replica == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
const SReplica *pReplica = pOption->replicas + i; const SReplica *pReplica = pOption->replicas + i;
if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1; if ((code = tjsonAddDoubleToObject(replica, "id", pReplica->id)) < 0) return code;
if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1; if ((code = tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn)) < 0) return code;
if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1; if ((code = tjsonAddDoubleToObject(replica, "port", pReplica->port)) < 0) return code;
if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1; if ((code = tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i])) < 0) return code;
if (tjsonAddItemToArray(replicas, replica) < 0) return -1; if ((code = tjsonAddItemToArray(replicas, replica)) < 0) return code;
} }
} }
if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1; if ((code = tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex)) < 0) return code;
if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1; if ((code = tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy)) < 0) return code;
return 0; return code;
} }
int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
@ -158,28 +167,59 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) {
if (mmEncodeOption(pJson, pOption) != 0) goto _OVER; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
TAOS_CHECK_GOTO(mmEncodeOption(pJson, pOption), NULL, _OVER);
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosCloseFile(&pFile) < 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy); dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy);
_OVER: _OVER:
@ -188,8 +228,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, tstrerror(code), pOption->deploy);
dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, terrstr(), pOption->deploy);
} }
return code; return code;
} }

37
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -26,11 +26,12 @@ void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) {
} }
int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SDCreateMnodeReq createReq = {0}; SDCreateMnodeReq createReq = {0};
if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
SMnodeOpt option = {.deploy = true, SMnodeOpt option = {.deploy = true,
@ -56,43 +57,45 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
if (option.selfIndex == -1) { if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s, selfIndex is -1", terrstr()); dGError("failed to create mnode since %s, selfIndex is -1", tstrerror(code));
return -1; return code;
} }
if (mmWriteFile(pInput->path, &option) != 0) { if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", terrstr()); dGError("failed to write mnode file since %s", tstrerror(code));
return -1; return code;
} }
return 0; return 0;
} }
int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SDDropMnodeReq dropReq = {0}; SDDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dGError("failed to drop mnode since %s", terrstr()); dGError("failed to drop mnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
SMnodeOpt option = {.deploy = false}; SMnodeOpt option = {.deploy = false};
if (mmWriteFile(pInput->path, &option) != 0) { if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", terrstr()); dGError("failed to write mnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return 0; return code;
} }
SArray *mmGetMsgHandles() { SArray *mmGetMsgHandles() {

68
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -25,9 +25,10 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
} }
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
int32_t code = 0;
SMnodeOpt option = {0}; SMnodeOpt option = {0};
if (mmReadFile(pInput->path, &option) != 0) { if ((code = mmReadFile(pInput->path, &option)) != 0) {
return -1; return code;
} }
if (!option.deploy) { if (!option.deploy) {
@ -41,7 +42,7 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
dInfo("deploy mnode required. option deploy:%d", option.deploy); dInfo("deploy mnode required. option deploy:%d", option.deploy);
} }
return 0; return code;
} }
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
@ -73,22 +74,31 @@ static void mmClose(SMnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
static int32_t mndOpenWrapper(const char *path, SMnodeOpt *opt, SMnode **pMnode) {
int32_t code = 0;
*pMnode = mndOpen(path, opt);
if (*pMnode == NULL) {
code = terrno;
}
///*pMnode = pNode;
return code;
}
static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (walInit() != 0) { int32_t code = 0;
dError("failed to init wal since %s", terrstr()); if ((code = walInit()) != 0) {
return -1; dError("failed to init wal since %s", tstrerror(code));
return code;
} }
if (syncInit() != 0) { if ((code = syncInit()) != 0) {
dError("failed to init sync since %s", terrstr()); dError("failed to init sync since %s", tstrerror(code));
return -1; return code;
} }
SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt)); SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -100,10 +110,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
taosThreadRwlockInit(&pMgmt->lock, NULL); taosThreadRwlockInit(&pMgmt->lock, NULL);
SMnodeOpt option = {0}; SMnodeOpt option = {0};
if (mmReadFile(pMgmt->path, &option) != 0) { if ((code = mmReadFile(pMgmt->path, &option)) != 0) {
dError("failed to read file since %s", terrstr()); dError("failed to read file since %s", tstrerror(code));
mmClose(pMgmt); mmClose(pMgmt);
return -1; return code;
} }
if (!option.deploy) { if (!option.deploy) {
@ -115,18 +125,18 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
mmBuildOptionForOpen(pMgmt, &option); mmBuildOptionForOpen(pMgmt, &option);
} }
pMgmt->pMnode = mndOpen(pMgmt->path, &option); code = mndOpenWrapper(pMgmt->path, &option, &pMgmt->pMnode);
if (pMgmt->pMnode == NULL) { if (code != 0) {
dError("failed to open mnode since %s", terrstr()); dError("failed to open mnode since %s", tstrerror(code));
mmClose(pMgmt); mmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("mnode-impl", "initialized"); tmsgReportStartup("mnode-impl", "initialized");
if (mmStartWorker(pMgmt) != 0) { if ((code = mmStartWorker(pMgmt)) != 0) {
dError("failed to start mnode worker since %s", terrstr()); dError("failed to start mnode worker since %s", tstrerror(code));
mmClose(pMgmt); mmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("mnode-worker", "initialized"); tmsgReportStartup("mnode-worker", "initialized");
@ -134,9 +144,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
option.deploy = true; option.deploy = true;
option.numOfReplicas = 0; option.numOfReplicas = 0;
option.numOfTotalReplicas = 0; option.numOfTotalReplicas = 0;
if (mmWriteFile(pMgmt->path, &option) != 0) { if ((code = mmWriteFile(pMgmt->path, &option)) != 0) {
dError("failed to write mnode file since %s", terrstr()); dError("failed to write mnode file since %s", tstrerror(code));
return -1; return code;
} }
} }
@ -160,13 +170,9 @@ static void mmStop(SMnodeMgmt *pMgmt) {
mndStop(pMgmt->pMnode); mndStop(pMgmt->pMnode);
} }
static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { return mndIsCatchUp(pMgmt->pMnode); }
return mndIsCatchUp(pMgmt->pMnode);
}
static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { return mndGetRole(pMgmt->pMnode); }
return mndGetRole(pMgmt->pMnode);
}
SMgmtFunc mmGetMgmtFunc() { SMgmtFunc mmGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0}; SMgmtFunc mgmtFunc = {0};

51
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -179,13 +179,15 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pWorker = &pMgmt->syncRdWorker; pWorker = &pMgmt->syncRdWorker;
break; break;
default: default:
terrno = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
} }
if (pWorker == NULL) return -1; if (pWorker == NULL) return code;
SRpcMsg *pMsg; SRpcMsg *pMsg;
code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg); code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) return code; if (code) return code;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL; pRpc->pCont = NULL;
@ -201,6 +203,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
} }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg qCfg = { SSingleWorkerCfg qCfg = {
.min = tsNumOfMnodeQueryThreads, .min = tsNumOfMnodeQueryThreads,
.max = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads,
@ -209,9 +212,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.param = pMgmt, .param = pMgmt,
.poolType = QUERY_AUTO_QWORKER_POOL, .poolType = QUERY_AUTO_QWORKER_POOL,
}; };
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &qCfg)) != 0) {
dError("failed to start mnode-query worker since %s", terrstr()); dError("failed to start mnode-query worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg fCfg = { SSingleWorkerCfg fCfg = {
@ -221,9 +224,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg)) != 0) {
dError("failed to start mnode-fetch worker since %s", terrstr()); dError("failed to start mnode-fetch worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg rCfg = { SSingleWorkerCfg rCfg = {
@ -233,9 +236,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->readWorker, &rCfg)) != 0) {
dError("failed to start mnode-read worker since %s", terrstr()); dError("failed to start mnode-read worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg wCfg = { SSingleWorkerCfg wCfg = {
@ -245,9 +248,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->writeWorker, &wCfg)) != 0) {
dError("failed to start mnode-write worker since %s", terrstr()); dError("failed to start mnode-write worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg sCfg = { SSingleWorkerCfg sCfg = {
@ -257,9 +260,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg, .fp = (FItem)mmProcessSyncMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->syncWorker, &sCfg)) != 0) {
dError("failed to start mnode mnode-sync worker since %s", terrstr()); dError("failed to start mnode mnode-sync worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg scCfg = { SSingleWorkerCfg scCfg = {
@ -269,9 +272,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg, .fp = (FItem)mmProcessSyncMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg)) != 0) {
dError("failed to start mnode mnode-sync-rd worker since %s", terrstr()); dError("failed to start mnode mnode-sync-rd worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg arbCfg = { SSingleWorkerCfg arbCfg = {
@ -281,13 +284,13 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg)) != 0) {
dError("failed to start mnode mnode-arb worker since %s", terrstr()); dError("failed to start mnode mnode-arb worker since %s", tstrerror(code));
return -1; return code;
} }
dDebug("mnode workers are initialized"); dDebug("mnode workers are initialized");
return 0; return code;
} }
void mmStopWorker(SMnodeMgmt *pMgmt) { void mmStopWorker(SMnodeMgmt *pMgmt) {

34
source/dnode/mgmt/mgmt_qnode/src/qmHandle.c
View File

@ -30,24 +30,25 @@ void qmGetQnodeLoads(SQnodeMgmt *pMgmt, SQnodeLoad *pInfo) {
} }
int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0}; SDCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to create qnode since %s", terrstr()); dError("failed to create qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
bool deployed = true; bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", terrstr()); dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
@ -55,24 +56,25 @@ int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0}; SDDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop qnode since %s", terrstr()); dError("failed to drop qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
bool deployed = false; bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", terrstr()); dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);

30
source/dnode/mgmt/mgmt_qnode/src/qmInt.c
View File

@ -33,11 +33,18 @@ static void qmClose(SQnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption);
if (*pQnode == NULL) {
return terrno;
}
return 0;
}
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt)); SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -50,29 +57,30 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SQnodeOpt option = {0}; SQnodeOpt option = {0};
qmInitOption(pMgmt, &option); qmInitOption(pMgmt, &option);
pMgmt->pQnode = qndOpen(&option);
if (pMgmt->pQnode == NULL) { code = qndOpenWrapper(&option, &pMgmt->pQnode);
dError("failed to open qnode since %s", terrstr()); if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt); qmClose(pMgmt);
return -1; return -1;
} }
tmsgReportStartup("qnode-impl", "initialized"); tmsgReportStartup("qnode-impl", "initialized");
if (udfcOpen() != 0) { if ((code = udfcOpen()) != 0) {
dError("qnode can not open udfc"); dError("qnode can not open udfc");
qmClose(pMgmt); qmClose(pMgmt);
return -1; return code;
} }
if (qmStartWorker(pMgmt) != 0) { if ((code = qmStartWorker(pMgmt)) != 0) {
dError("failed to start qnode worker since %s", terrstr()); dError("failed to start qnode worker since %s", tstrerror(code));
qmClose(pMgmt); qmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("qnode-worker", "initialized"); tmsgReportStartup("qnode-worker", "initialized");
pOutput->pMgmt = pMgmt; pOutput->pMgmt = pMgmt;
return 0; return code;
} }
SMgmtFunc qmGetMgmtFunc() { SMgmtFunc qmGetMgmtFunc() {

16
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -102,6 +102,8 @@ int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
} }
int32_t qmStartWorker(SQnodeMgmt *pMgmt) { int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg queryCfg = { SSingleWorkerCfg queryCfg = {
.min = tsNumOfVnodeQueryThreads, .min = tsNumOfVnodeQueryThreads,
.max = tsNumOfVnodeQueryThreads, .max = tsNumOfVnodeQueryThreads,
@ -111,9 +113,9 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.poolType = QUERY_AUTO_QWORKER_POOL, .poolType = QUERY_AUTO_QWORKER_POOL,
}; };
if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg)) != 0) {
dError("failed to start qnode-query worker since %s", terrstr()); dError("failed to start qnode-query worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg fetchCfg = { SSingleWorkerCfg fetchCfg = {
@ -124,13 +126,13 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg)) != 0) {
dError("failed to start qnode-fetch worker since %s", terrstr()); dError("failed to start qnode-fetch worker since %s", tstrerror(code));
return -1; return code;
} }
dDebug("qnode workers are initialized"); dDebug("qnode workers are initialized");
return 0; return code;
} }
void qmStopWorker(SQnodeMgmt *pMgmt) { void qmStopWorker(SQnodeMgmt *pMgmt) {

35
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -19,24 +19,25 @@
void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {}
int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0}; SDCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to create snode since %s", terrstr()); dError("failed to create snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
bool deployed = true; bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", terrstr()); dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
@ -44,24 +45,26 @@ int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1;
return code;
} }
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop snode since %s", terrstr()); dError("failed to drop snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
bool deployed = false; bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", terrstr()); dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);

38
source/dnode/mgmt/mgmt_snode/src/smInt.c
View File

@ -33,12 +33,19 @@ static void smClose(SSnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
int32_t sndOpenWrapper(const char *path, SSnodeOpt *pOption, SSnode **pNode) {
*pNode = sndOpen(path, pOption);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt)); SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -50,35 +57,34 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SSnodeOpt option = {0}; SSnodeOpt option = {0};
smInitOption(pMgmt, &option); smInitOption(pMgmt, &option);
pMgmt->pSnode = sndOpen(pMgmt->path, &option);
if (pMgmt->pSnode == NULL) { code = sndOpenWrapper(pMgmt->path, &option, &pMgmt->pSnode);
dError("failed to open snode since %s", terrstr()); if (code != 0) {
dError("failed to open snode since %s", tstrerror(code));
smClose(pMgmt); smClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("snode-impl", "initialized"); tmsgReportStartup("snode-impl", "initialized");
if (smStartWorker(pMgmt) != 0) { if ((code = smStartWorker(pMgmt)) != 0) {
dError("failed to start snode worker since %s", terrstr()); dError("failed to start snode worker since %s", tstrerror(code));
smClose(pMgmt); smClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("snode-worker", "initialized"); tmsgReportStartup("snode-worker", "initialized");
if (udfcOpen() != 0) { if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in snode"); dError("failed to open udfc in snode since:%s", tstrerror(code));
smClose(pMgmt); smClose(pMgmt);
return -1; return code;
} }
pOutput->pMgmt = pMgmt; pOutput->pMgmt = pMgmt;
return 0; return 0;
} }
static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { return sndInit(pMgmt->pSnode); }
return sndInit(pMgmt->pSnode);
}
SMgmtFunc smGetMgmtFunc() { SMgmtFunc smGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0}; SMgmtFunc mgmtFunc = {0};

60
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -68,17 +68,18 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
} }
int32_t smStartWorker(SSnodeMgmt *pMgmt) { int32_t smStartWorker(SSnodeMgmt *pMgmt) {
int32_t code = 0;
pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *)); pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->writeWroker == NULL) { if (pMgmt->writeWroker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) { for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) {
SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker)); SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pWriteWorker == NULL) { if (pWriteWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
SMultiWorkerCfg cfg = { SMultiWorkerCfg cfg = {
@ -87,13 +88,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.fp = smProcessWriteQueue, .fp = smProcessWriteQueue,
.param = pMgmt, .param = pMgmt,
}; };
if (tMultiWorkerInit(pWriteWorker, &cfg) != 0) { if ((code = tMultiWorkerInit(pWriteWorker, &cfg)) != 0) {
dError("failed to start snode-unique worker since %s", terrstr()); dError("failed to start snode-unique worker since %s", tstrerror(code));
return -1; return code;
} }
if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) { if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
} }
@ -105,13 +106,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) { if ((code = tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) != 0) {
dError("failed to start snode shared-worker since %s", terrstr()); dError("failed to start snode shared-worker since %s", tstrerror(code));
return -1; return code;
} }
dDebug("snode workers are initialized"); dDebug("snode workers are initialized");
return 0; return code;
} }
void smStopWorker(SSnodeMgmt *pMgmt) { void smStopWorker(SSnodeMgmt *pMgmt) {
@ -133,17 +134,18 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) { if (code) {
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return code = terrno;
} }
SSnode *pSnode = pMgmt->pSnode; SSnode *pSnode = pMgmt->pSnode;
if (pSnode == NULL) { if (pSnode == NULL) {
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, terrstr(), code = terrno;
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, tstrerror(code),
TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen); TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return code;
} }
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
@ -154,48 +156,44 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
switch (qtype) { switch (qtype) {
case STREAM_QUEUE: case STREAM_QUEUE:
smPutNodeMsgToStreamQueue(pMgmt, pMsg); code = smPutNodeMsgToStreamQueue(pMgmt, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
smPutNodeMsgToWriteQueue(pMgmt, pMsg); code = smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break; break;
default: default:
terrno = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
return -1; return code;
} }
return 0; return code;
} }
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->streamWorker; SSingleWorker *pWorker = &pMgmt->streamWorker;
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }

2
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -109,7 +109,7 @@ int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
// vmFile.c // vmFile.c
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt); int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes); int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
// vmWorker.c // vmWorker.c
int32_t vmStartWorker(SVnodeMgmt *pMgmt); int32_t vmStartWorker(SVnodeMgmt *pMgmt);

141
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -19,12 +19,16 @@
#define MAX_CONTENT_LEN 2 * 1024 * 1024 #define MAX_CONTENT_LEN 2 * 1024 * 1024
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
taosThreadRwlockRdlock(&pMgmt->lock); taosThreadRwlockRdlock(&pMgmt->lock);
int32_t num = 0; int32_t num = 0;
int32_t size = taosHashGetSize(pMgmt->hash); int32_t size = taosHashGetSize(pMgmt->hash);
SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *)); SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
if (pVnodes == NULL) {
taosThreadRwlockUnlock(&pMgmt->lock);
return TSDB_CODE_OUT_OF_MEMORY;
}
void *pIter = taosHashIterate(pMgmt->hash, NULL); void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) { while (pIter) {
@ -42,8 +46,9 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
taosThreadRwlockUnlock(&pMgmt->lock); taosThreadRwlockUnlock(&pMgmt->lock);
*numOfVnodes = num; *numOfVnodes = num;
*ppVnodes = pVnodes;
return pVnodes; return 0;
} }
static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
@ -52,29 +57,32 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **
*ppCfgs = NULL; *ppCfgs = NULL;
SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes"); SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes");
if (vnodes == NULL) return -1; if (vnodes == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t vnodesNum = cJSON_GetArraySize(vnodes); int32_t vnodesNum = cJSON_GetArraySize(vnodes);
if (vnodesNum > 0) { if (vnodesNum > 0) {
pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg));
if (pCfgs == NULL) return -1; if (pCfgs == NULL) return TSDB_CODE_OUT_OF_MEMORY;
} }
for (int32_t i = 0; i < vnodesNum; ++i) { for (int32_t i = 0; i < vnodesNum; ++i) {
SJson *vnode = tjsonGetArrayItem(vnodes, i); SJson *vnode = tjsonGetArrayItem(vnodes, i);
if (vnode == NULL) goto _OVER; if (vnode == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
SWrapperCfg *pCfg = &pCfgs[i]; SWrapperCfg *pCfg = &pCfgs[i];
tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code); tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code); tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code); tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code); tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code); tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
} }
@ -98,33 +106,35 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file); code = TAOS_SYSTEM_ERROR(errno);
return 0; dInfo("vnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0;
return code;
} }
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open vnode file:%s since %s", file, terrstr()); dError("failed to open vnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr()); dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
pData = taosMemoryMalloc(size + 1); pData = taosMemoryMalloc(size + 1);
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, pData, size) != size) { if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read vnode file:%s since %s", file, terrstr()); dError("failed to read vnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -132,12 +142,12 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
pJson = tjsonParse(pData); pJson = tjsonParse(pData);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) { if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -150,28 +160,36 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read vnode file:%s since %s", file, terrstr()); dError("failed to read vnode file:%s since %s", file, tstrerror(code));
} }
return code; return code;
} }
static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) { static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) {
int32_t code = 0;
SJson *vnodes = tjsonCreateArray(); SJson *vnodes = tjsonCreateArray();
if (vnodes == NULL) return -1; if (vnodes == NULL) {
if (tjsonAddItemToObject(pJson, "vnodes", vnodes) < 0) return -1; return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "vnodes", vnodes)) < 0) {
tjsonDelete(vnodes);
return code;
};
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i]; SVnodeObj *pVnode = ppVnodes[i];
if (pVnode == NULL) continue; if (pVnode == NULL) continue;
SJson *vnode = tjsonCreateObject(); SJson *vnode = tjsonCreateObject();
if (vnode == NULL) return -1; if (vnode == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId)) < 0) return code;
if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped)) < 0) return code;
if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion)) < 0) return code;
if (tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary)) < 0) return code;
if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1; if (pVnode->toVgId) {
if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId)) < 0) return code;
}
if ((code = tjsonAddItemToArray(vnodes, vnode)) < 0) return code;
} }
return 0; return 0;
@ -185,30 +203,60 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = NULL; SVnodeObj **ppVnodes = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); int32_t nBytes = snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
return TSDB_CODE_OUT_OF_RANGE;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (ppVnodes == NULL) goto _OVER; if (code) goto _OVER;
terrno = TSDB_CODE_OUT_OF_MEMORY; // terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) {
if (vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes) != 0) goto _OVER; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER;
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); code = taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER; if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0; code = 0;
dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes); dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes);
@ -228,8 +276,7 @@ _OVER:
} }
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, tstrerror(code), numOfVnodes);
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes);
} }
return code; return code;
} }

33
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -256,8 +256,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
char path[TSDB_FILENAME_LEN] = {0}; char path[TSDB_FILENAME_LEN] = {0};
if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) { if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
if (req.learnerReplica == 0) { if (req.learnerReplica == 0) {
@ -298,25 +297,24 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort ||
strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { strcmp(pReplica->fqdn, tsLocalFqdn) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", req.vgId, pReplica->id, pReplica->fqdn, dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode, reason:%s", req.vgId, pReplica->id,
pReplica->port); pReplica->fqdn, pReplica->port, tstrerror(code));
return -1; return code;
} }
if (req.encryptAlgorithm == DND_CA_SM4) { if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) { if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId); dError("vgId:%d, failed to create vnode since encrypt key is empty, reason:%s", req.vgId, tstrerror(code));
return -1; return code;
} }
} }
vmGenerateVnodeCfg(&req, &vnodeCfg); vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { if ((code = vmTsmaAdjustDays(&vnodeCfg, &req)) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr()); dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, tstrerror(code));
code = terrno != 0 ? terrno : -1;
goto _OVER; goto _OVER;
} }
@ -327,8 +325,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("vgId:%d, already exist", req.vgId); dError("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req); tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_VND_ALREADY_EXIST; code = TSDB_CODE_VND_ALREADY_EXIST;
code = terrno;
return 0; return 0;
} }
@ -377,14 +374,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER; goto _OVER;
} }
taosThreadMutexLock(&pMgmt->createLock); //taosThreadMutexLock(&pMgmt->createLock);
code = vmWriteVnodeListToFile(pMgmt); code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) { if (code != 0) {
code = terrno != 0 ? terrno : code; code = terrno != 0 ? terrno : code;
taosThreadMutexUnlock(&pMgmt->createLock); //taosThreadMutexUnlock(&pMgmt->createLock);
goto _OVER; goto _OVER;
} }
taosThreadMutexUnlock(&pMgmt->createLock); //taosThreadMutexUnlock(&pMgmt->createLock);
_OVER: _OVER:
if (code != 0) { if (code != 0) {
@ -988,7 +985,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMultiMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

101
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -33,6 +33,7 @@ int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
} }
int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t code = 0;
STfs *pTfs = pMgmt->pTfs; STfs *pTfs = pMgmt->pTfs;
int32_t diskId = 0; int32_t diskId = 0;
if (!pTfs) { if (!pTfs) {
@ -59,7 +60,12 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
// alloc // alloc
int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0}; int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0};
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
return code;
}
for (int32_t v = 0; v < numOfVnodes; v++) { for (int32_t v = 0; v < numOfVnodes; v++) {
SVnodeObj *pVnode = ppVnodes[v]; SVnodeObj *pVnode = ppVnodes[v];
disks[pVnode->diskPrimary] += 1; disks[pVnode->diskPrimary] += 1;
@ -436,6 +442,7 @@ static void *vmCloseVnodeInThread(void *param) {
} }
static void vmCloseVnodes(SVnodeMgmt *pMgmt) { static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
dInfo("start to close all vnodes"); dInfo("start to close all vnodes");
tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dInfo("vnodes mgmt worker is stopped"); dInfo("vnodes mgmt worker is stopped");
@ -443,7 +450,12 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
dInfo("vnodes multiple mgmt worker is stopped"); dInfo("vnodes multiple mgmt worker is stopped");
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
int32_t threadNum = tsNumOfCores / 2; int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1; if (threadNum < 1) threadNum = 1;
@ -513,8 +525,14 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
} }
static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
if (ppVnodes != NULL) { if (ppVnodes != NULL) {
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
@ -549,12 +567,14 @@ static void *vmThreadFp(void *param) {
} }
static int32_t vmInitTimer(SVnodeMgmt *pMgmt) { static int32_t vmInitTimer(SVnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) {
dError("failed to create vnode timer thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create vnode timer thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -573,7 +593,10 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = -1; int32_t code = -1;
SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt)); SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt));
if (pMgmt == NULL) goto _OVER; if (pMgmt == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
pMgmt->path = pInput->path; pMgmt->path = pInput->path;
@ -582,8 +605,18 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue; pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue;
pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize; pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize;
pMgmt->msgCb.mgmt = pMgmt; pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
taosThreadMutexInit(&pMgmt->createLock, NULL); code = taosThreadRwlockInit(&pMgmt->lock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = taosThreadMutexInit(&pMgmt->createLock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
pMgmt->pTfs = pInput->pTfs; pMgmt->pTfs = pInput->pTfs;
if (pMgmt->pTfs == NULL) { if (pMgmt->pTfs == NULL) {
@ -592,38 +625,39 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
} }
tmsgReportStartup("vnode-tfs", "initialized"); tmsgReportStartup("vnode-tfs", "initialized");
if (walInit() != 0) { if ((code = walInit()) != 0) {
dError("failed to init wal since %s", terrstr()); dError("failed to init wal since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-wal", "initialized"); tmsgReportStartup("vnode-wal", "initialized");
if (syncInit() != 0) { if ((code = syncInit()) != 0) {
dError("failed to open sync since %s", terrstr()); dError("failed to open sync since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-sync", "initialized"); tmsgReportStartup("vnode-sync", "initialized");
if (vnodeInit(tsNumOfCommitThreads) != 0) { if ((code = vnodeInit(tsNumOfCommitThreads)) != 0) {
dError("failed to init vnode since %s", terrstr()); dError("failed to init vnode since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-commit", "initialized"); tmsgReportStartup("vnode-commit", "initialized");
if (vmStartWorker(pMgmt) != 0) { if ((code = vmStartWorker(pMgmt)) != 0) {
dError("failed to init workers since %s", terrstr()); dError("failed to init workers since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-worker", "initialized"); tmsgReportStartup("vnode-worker", "initialized");
if (vmOpenVnodes(pMgmt) != 0) { if ((code = vmOpenVnodes(pMgmt)) != 0) {
dError("failed to open all vnodes since %s", terrstr()); dError("failed to open all vnodes since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-vnodes", "initialized"); tmsgReportStartup("vnode-vnodes", "initialized");
if (udfcOpen() != 0) { if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in vnode"); dError("failed to open udfc in vnode since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -633,7 +667,7 @@ _OVER:
if (code == 0) { if (code == 0) {
pOutput->pMgmt = pMgmt; pOutput->pMgmt = pMgmt;
} else { } else {
dError("failed to init vnodes-mgmt since %s", terrstr()); dError("failed to init vnodes-mgmt since %s", tstrerror(code));
vmCleanup(pMgmt); vmCleanup(pMgmt);
} }
@ -683,18 +717,32 @@ static void *vmRestoreVnodeInThread(void *param) {
} }
static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return code;
}
int32_t threadNum = tsNumOfCores / 2; int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1; if (threadNum < 1) threadNum = 1;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1; int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread));
if (threads == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t t = 0; t < threadNum; ++t) { for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t; threads[t].threadIndex = t;
threads[t].pMgmt = pMgmt; threads[t].pMgmt = pMgmt;
threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *)); threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *));
if (threads[t].ppVnodes == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
} }
for (int32_t v = 0; v < numOfVnodes; ++v) { for (int32_t v = 0; v < numOfVnodes; ++v) {
@ -717,6 +765,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) { if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) {
dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno)); dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno));
ASSERT(errno == 0);
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -742,6 +791,14 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
} }
return vmInitTimer(pMgmt); return vmInitTimer(pMgmt);
_exit:
for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t];
taosMemoryFree(pThread->ppVnodes);
}
taosMemoryFree(threads);
return code;
} }
static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); } static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); }

70
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -200,26 +200,33 @@ static bool vmDataSpaceSufficient(SVnodeObj *pVnode) {
} }
} }
static int32_t vmAcquireVnodeWrapper(SVnodeMgmt *pMgt, int32_t vgId, SVnodeObj **pNode) {
*pNode = vmAcquireVnode(pMgt, vgId);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
if (pMsg->contLen < sizeof(SMsgHead)) { if (pMsg->contLen < sizeof(SMsgHead)) {
dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType), dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType),
pMsg->contLen); pMsg->contLen);
return -1; return TSDB_CODE_INVALID_MSG;
} }
SMsgHead *pHead = pMsg->pCont; SMsgHead *pHead = pMsg->pCont;
int32_t code = 0;
pHead->contLen = ntohl(pHead->contLen); pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId); pHead->vgId = ntohl(pHead->vgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = NULL;
if (pVnode == NULL) { code = vmAcquireVnodeWrapper(pMgmt, pHead->vgId, &pVnode);
if (code != 0) {
dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); tstrerror(code), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
terrno = (terrno != 0) ? terrno : -1; return code;
return terrno;
} }
switch (qtype) { switch (qtype) {
@ -234,49 +241,45 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
break; break;
case STREAM_QUEUE: case STREAM_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pStreamQ, pMsg); code = taosWriteQitem(pVnode->pStreamQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pFetchQ, pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
if (!vmDataSpaceSufficient(pVnode)) { if (!vmDataSpaceSufficient(pVnode)) {
terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE; code = TSDB_CODE_NO_ENOUGH_DISKSPACE;
code = terrno;
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code)); dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break; break;
} }
if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) {
terrno = TSDB_CODE_VND_NO_WRITE_AUTH; code = TSDB_CODE_VND_NO_WRITE_AUTH;
code = terrno;
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code)); dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break; break;
} }
if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) { if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) {
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg); dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg);
terrno = TSDB_CODE_VND_STOPPED; code = TSDB_CODE_VND_STOPPED;
code = terrno;
break; break;
} }
dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pWriteW.queue, pMsg); code = taosWriteQitem(pVnode->pWriteW.queue, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncW.queue, pMsg); code = taosWriteQitem(pVnode->pSyncW.queue, pMsg);
break; break;
case SYNC_RD_QUEUE: case SYNC_RD_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncRdW.queue, pMsg); code = taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
break; break;
case APPLY_QUEUE: case APPLY_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pApplyW.queue, pMsg); code = taosWriteQitem(pVnode->pApplyW.queue, pMsg);
break; break;
default: default:
code = -1; code = TSDB_CODE_INVALID_MSG;
terrno = TSDB_CODE_INVALID_PARA;
break; break;
} }
@ -299,15 +302,13 @@ int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMs
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg); dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg); return taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
return 0;
} }
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-mgmt queue", pMsg); dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg); return taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg);
return 0;
} }
int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
@ -317,7 +318,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pRpc->contLen); pRpc->contLen);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return TSDB_CODE_INVALID_MSG;
} }
EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM; EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM;
@ -326,7 +327,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) { if (code) {
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return code;
} }
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
@ -383,6 +384,7 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
} }
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
int32_t code = 0;
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl}; SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
@ -398,8 +400,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL || if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL ||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) { pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
@ -428,26 +429,27 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
} }
int32_t vmStartWorker(SVnodeMgmt *pMgmt) { int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
int32_t code = 0;
SQueryAutoQWorkerPool *pQPool = &pMgmt->queryPool; SQueryAutoQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query"; pQPool->name = "vnode-query";
pQPool->min = tsNumOfVnodeQueryThreads; pQPool->min = tsNumOfVnodeQueryThreads;
pQPool->max = tsNumOfVnodeQueryThreads; pQPool->max = tsNumOfVnodeQueryThreads;
if (tQueryAutoQWorkerInit(pQPool) != 0) return -1; if ((code = tQueryAutoQWorkerInit(pQPool)) != 0) return code;
SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool; SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool;
pStreamPool->name = "vnode-stream"; pStreamPool->name = "vnode-stream";
pStreamPool->ratio = tsRatioOfVnodeStreamThreads; pStreamPool->ratio = tsRatioOfVnodeStreamThreads;
if (tAutoQWorkerInit(pStreamPool) != 0) return -1; if ((code = tAutoQWorkerInit(pStreamPool)) != 0) return code;
SWWorkerPool *pFPool = &pMgmt->fetchPool; SWWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch"; pFPool->name = "vnode-fetch";
pFPool->max = tsNumOfVnodeFetchThreads; pFPool->max = tsNumOfVnodeFetchThreads;
if (tWWorkerInit(pFPool) != 0) return -1; if ((code = tWWorkerInit(pFPool)) != 0) return code;
SSingleWorkerCfg mgmtCfg = { SSingleWorkerCfg mgmtCfg = {
.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; .min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1; if ((code = tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg)) != 0) return code;
int32_t threadNum = 0; int32_t threadNum = 0;
if (tsNumOfCores == 1) { if (tsNumOfCores == 1) {
@ -461,7 +463,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
.fp = (FItem)vmProcessMultiMgmtQueue, .fp = (FItem)vmProcessMultiMgmtQueue,
.param = pMgmt}; .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg) != 0) return -1; if ((code = tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg)) != 0) return code;
dDebug("vnode workers are initialized"); dDebug("vnode workers are initialized");
return 0; return 0;

1
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -98,6 +98,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType);
int32_t dmMarkWrapper(SMgmtWrapper *pWrapper); int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper); void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode); int32_t dmInitVars(SDnode *pDnode);
int32_t dmInitVarsWrapper(SDnode *pDnode);
void dmClearVars(SDnode *pDnode); void dmClearVars(SDnode *pDnode);
int32_t dmInitModule(SDnode *pDnode); int32_t dmInitModule(SDnode *pDnode);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);

145
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -14,19 +14,21 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
// clang-format off
#include "dmMgmt.h" #include "dmMgmt.h"
#include "audit.h" #include "audit.h"
#include "libs/function/tudf.h" #include "libs/function/tudf.h"
#include "tgrant.h" #include "tgrant.h"
#include "tcompare.h" #include "tcompare.h"
// clang-format on
#define DM_INIT_AUDIT() \ #define DM_INIT_AUDIT() \
do { \ do { \
auditCfg.port = tsMonitorPort; \ auditCfg.port = tsMonitorPort; \
auditCfg.server = tsMonitorFqdn; \ auditCfg.server = tsMonitorFqdn; \
auditCfg.comp = tsMonitorComp; \ auditCfg.comp = tsMonitorComp; \
if (auditInit(&auditCfg) != 0) { \ if ((code = auditInit(&auditCfg)) != 0) { \
return -1; \ return code; \
} \ } \
} while (0) } while (0)
@ -35,10 +37,11 @@ static SDnode globalDnode = {0};
SDnode *dmInstance() { return &globalDnode; } SDnode *dmInstance() { return &globalDnode; }
static int32_t dmCheckRepeatInit(SDnode *pDnode) { static int32_t dmCheckRepeatInit(SDnode *pDnode) {
int32_t code = 0;
if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
dError("env is already initialized"); dError("env is already initialized");
terrno = TSDB_CODE_REPEAT_INIT; code = TSDB_CODE_REPEAT_INIT;
return -1; return code;
} }
return 0; return 0;
} }
@ -58,18 +61,15 @@ static int32_t dmInitMonitor() {
monCfg.port = tsMonitorPort; monCfg.port = tsMonitorPort;
monCfg.server = tsMonitorFqdn; monCfg.server = tsMonitorFqdn;
monCfg.comp = tsMonitorComp; monCfg.comp = tsMonitorComp;
if (monInit(&monCfg) != 0) { if ((code = monInit(&monCfg)) != 0) {
if (terrno != 0) code = terrno; dError("failed to init monitor since %s", tstrerror(code));
goto _exit;
} }
_exit:
if (code) terrno = code;
return code; return code;
} }
static int32_t dmInitAudit() { static int32_t dmInitAudit() {
SAuditCfg auditCfg = {0}; SAuditCfg auditCfg = {0};
int32_t code = 0;
DM_INIT_AUDIT(); DM_INIT_AUDIT();
@ -88,29 +88,35 @@ static bool dmDataSpaceAvailable() {
return true; return true;
} }
static bool dmCheckDiskSpace() { static int32_t dmCheckDiskSpace() {
osUpdate(); osUpdate();
// availability // availability
bool ret = true; int32_t code = 0;
if (!dmDataSpaceAvailable()) { if (!dmDataSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE; code = TSDB_CODE_NO_DISKSPACE;
ret = false; return code;
} }
if (!osLogSpaceAvailable()) { if (!osLogSpaceAvailable()) {
dError("log disk space unavailable, i.e. %s", tsLogDir); dError("log disk space unavailable, i.e. %s", tsLogDir);
terrno = TSDB_CODE_NO_DISKSPACE; code = TSDB_CODE_NO_DISKSPACE;
ret = false; return code;
} }
if (!osTempSpaceAvailable()) { if (!osTempSpaceAvailable()) {
dError("temp disk space unavailable, i.e. %s", tsTempDir); dError("temp disk space unavailable, i.e. %s", tsTempDir);
terrno = TSDB_CODE_NO_DISKSPACE; code = TSDB_CODE_NO_DISKSPACE;
ret = false; return code;
} }
return ret; return code;
} }
int32_t tfsOpenWrapper(SDiskCfg *pCfg, int32_t ndisk, STfs **tfs) {
*tfs = tfsOpen(pCfg, ndisk);
if (*tfs == NULL) {
return terrno;
}
return 0;
}
int32_t dmDiskInit() { int32_t dmDiskInit() {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0}; SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN); tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
@ -121,8 +127,8 @@ int32_t dmDiskInit() {
numOfDisks = 1; numOfDisks = 1;
} }
code = tfsOpen(pDisks, numOfDisks, &pDnode->pTfs); int32_t code = tfsOpenWrapper(pDisks, numOfDisks, &pDnode->pTfs);
if (pDnode->pTfs == NULL) { if (code != 0) {
dError("failed to init tfs since %s", tstrerror(code)); dError("failed to init tfs since %s", tstrerror(code));
TAOS_RETURN(code); TAOS_RETURN(code);
} }
@ -147,6 +153,12 @@ static bool dmCheckDataDirVersion() {
return true; return true;
} }
static int32_t dmCheckDataDirVersionWrapper() {
if (!dmCheckDataDirVersion()) {
return TSDB_CODE_INVALID_DATA_FMT;
}
return 0;
}
#if defined(USE_S3) #if defined(USE_S3)
extern int32_t s3Begin(); extern int32_t s3Begin();
@ -157,17 +169,21 @@ extern int8_t tsS3Enabled;
int32_t dmInit() { int32_t dmInit() {
dInfo("start to init dnode env"); dInfo("start to init dnode env");
if (dmDiskInit() != 0) return -1; int32_t code = 0;
if (!dmCheckDataDirVersion()) return -1; if ((code = dmDiskInit()) != 0) return code;
if (!dmCheckDiskSpace()) return -1; if (!dmCheckDataDirVersion()) {
if (dmCheckRepeatInit(dmInstance()) != 0) return -1; code = TSDB_CODE_INVALID_DATA_FMT;
if (dmInitSystem() != 0) return -1; return code;
if (dmInitMonitor() != 0) return -1; }
if (dmInitAudit() != 0) return -1; if ((code = dmCheckDiskSpace()) != 0) return code;
if (dmInitDnode(dmInstance()) != 0) return -1; if ((code = dmCheckRepeatInit(dmInstance())) != 0) return code;
if (InitRegexCache() != 0) return -1; if ((code = dmInitSystem()) != 0) return code;
if ((code = dmInitMonitor()) != 0) return code;
if ((code = dmInitAudit()) != 0) return code;
if ((code = dmInitDnode(dmInstance())) != 0) return code;
if ((code = InitRegexCache() != 0)) return code;
#if defined(USE_S3) #if defined(USE_S3)
if (s3Begin() != 0) return -1; if ((code = s3Begin()) != 0) return code;
#endif #endif
dInfo("dnode env is initialized"); dInfo("dnode env is initialized");
@ -218,6 +234,7 @@ int32_t dmRun() {
} }
static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
@ -225,19 +242,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
switch (ntype) { switch (ntype) {
case MNODE: case MNODE:
terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED; code = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
break; break;
case QNODE: case QNODE:
terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED; code = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
break; break;
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED; code = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
break; break;
default: default:
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
} }
dError("failed to create node since %s", terrstr()); dError("failed to create node since %s", tstrerror(code));
return -1; return code;
} }
dInfo("start to process create-node-request"); dInfo("start to process create-node-request");
@ -245,18 +262,18 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return -1; return code;
} }
taosThreadMutexLock(&pDnode->mutex); taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name); dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg); code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) { if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else { } else {
dInfo("node:%s, has been created", pWrapper->name); dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper); code = dmOpenNode(pWrapper);
@ -272,12 +289,13 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
} }
static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) { if (pWrapper == NULL) {
dError("fail to process alter node type since node not exist"); dError("fail to process alter node type since node not exist");
return -1; return TSDB_CODE_INVALID_MSG;
} }
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
@ -290,16 +308,16 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dInfo("node:%s, checking node role:%d", pWrapper->name, role); dInfo("node:%s, checking node role:%d", pWrapper->name, role);
if (role == TAOS_SYNC_ROLE_VOTER) { if (role == TAOS_SYNC_ROLE_VOTER) {
dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role); dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role);
terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER; code = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
return -1; return code;
} }
} }
if (pWrapper->func.isCatchUpFp != NULL) { if (pWrapper->func.isCatchUpFp != NULL) {
dInfo("node:%s, checking node catch up", pWrapper->name); dInfo("node:%s, checking node catch up", pWrapper->name);
if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) { if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) {
terrno = TSDB_CODE_MNODE_NOT_CATCH_UP; code = TSDB_CODE_MNODE_NOT_CATCH_UP;
return -1; return code;
} }
} }
@ -315,17 +333,17 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex); taosThreadMutexUnlock(&pDnode->mutex);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return -1; return code;
} }
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name); dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg); code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) { if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else { } else {
dInfo("node:%s, has been created", pWrapper->name); dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper); code = dmOpenNode(pWrapper);
@ -341,35 +359,36 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
} }
static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) { if (pWrapper == NULL) {
switch (ntype) { switch (ntype) {
case MNODE: case MNODE:
terrno = TSDB_CODE_MNODE_NOT_DEPLOYED; code = TSDB_CODE_MNODE_NOT_DEPLOYED;
break; break;
case QNODE: case QNODE:
terrno = TSDB_CODE_QNODE_NOT_DEPLOYED; code = TSDB_CODE_QNODE_NOT_DEPLOYED;
break; break;
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_NOT_DEPLOYED; code = TSDB_CODE_SNODE_NOT_DEPLOYED;
break; break;
default: default:
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
} }
dError("failed to drop node since %s", terrstr()); dError("failed to drop node since %s", tstrerror(code));
return -1; return terrno = code;
} }
taosThreadMutexLock(&pDnode->mutex); taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to drop", pWrapper->name); dInfo("node:%s, start to drop", pWrapper->name);
int32_t code = (*pWrapper->func.dropFp)(&input, pMsg); code = (*pWrapper->func.dropFp)(&input, pMsg);
if (code != 0) { if (code != 0) {
dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); dError("node:%s, failed to drop since %s", pWrapper->name, tstrerror(code));
} else { } else {
dInfo("node:%s, has been dropped", pWrapper->name); dInfo("node:%s, has been dropped", pWrapper->name);
pWrapper->required = false; pWrapper->required = false;
@ -417,6 +436,4 @@ void dmReportStartup(const char *pName, const char *pDesc) {
int64_t dmGetClusterId() { return globalDnode.data.clusterId; } int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
bool dmReadyForTest() { bool dmReadyForTest() { return dmInstance()->data.dnodeVer > 0; }
return dmInstance()->data.dnodeVer > 0;
}

54
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -42,7 +42,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
int32_t code = -1; int32_t code = -1;
char path[PATH_MAX + 100] = {0}; char path[PATH_MAX + 100] = {0};
if (dmInitVars(pDnode) != 0) { if ((code = dmInitVarsWrapper(pDnode)) != 0) {
goto _OVER; goto _OVER;
} }
@ -65,26 +65,31 @@ int32_t dmInitDnode(SDnode *pDnode) {
snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name); snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = taosStrdup(path); pWrapper->path = taosStrdup(path);
if (pWrapper->path == NULL) { if (pWrapper->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
pWrapper->required = dmRequireNode(pDnode, pWrapper); pWrapper->required = dmRequireNode(pDnode, pWrapper);
} }
pDnode->lockfile = dmCheckRunning(tsDataDir); code = dmCheckRunning(tsDataDir, &pDnode->lockfile);
if (pDnode->lockfile == NULL) { if (code != 0) {
goto _OVER; goto _OVER;
} }
if (dmInitModule(pDnode) != 0) {
if ((code = dmInitModule(pDnode)) != 0) {
goto _OVER; goto _OVER;
} }
indexInit(tsNumOfCommitThreads); indexInit(tsNumOfCommitThreads);
streamMetaInit(); streamMetaInit();
dmInitStatusClient(pDnode); if ((code = dmInitStatusClient(pDnode)) != 0) {
dmInitSyncClient(pDnode); goto _OVER;
}
if ((code = dmInitSyncClient(pDnode)) != 0) {
goto _OVER;
}
dmReportStartup("dnode-transport", "initialized"); dmReportStartup("dnode-transport", "initialized");
dDebug("dnode is created, ptr:%p", pDnode); dDebug("dnode is created, ptr:%p", pDnode);
@ -94,7 +99,7 @@ _OVER:
if (code != 0 && pDnode != NULL) { if (code != 0 && pDnode != NULL) {
dmClearVars(pDnode); dmClearVars(pDnode);
pDnode = NULL; pDnode = NULL;
dError("failed to create dnode since %s", terrstr()); dError("failed to create dnode since %s", tstrerror(code));
} }
return code; return code;
@ -122,7 +127,15 @@ void dmCleanupDnode(SDnode *pDnode) {
dDebug("dnode is closed, ptr:%p", pDnode); dDebug("dnode is closed, ptr:%p", pDnode);
} }
int32_t dmInitVarsWrapper(SDnode *pDnode) {
int32_t code = dmInitVars(pDnode);
if (code == -1) {
return terrno;
}
return 0;
}
int32_t dmInitVars(SDnode *pDnode) { int32_t dmInitVars(SDnode *pDnode) {
int32_t code = 0;
SDnodeData *pData = &pDnode->data; SDnodeData *pData = &pDnode->data;
pData->dnodeId = 0; pData->dnodeId = 0;
pData->clusterId = 0; pData->clusterId = 0;
@ -138,21 +151,21 @@ int32_t dmInitVars(SDnode *pDnode) {
taosMemoryFreeClear(machineId); taosMemoryFreeClear(machineId);
} else { } else {
#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) #if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; code = TSDB_CODE_DNODE_NO_MACHINE_CODE;
return -1; return terrno = code;
#endif #endif
} }
pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pData->dnodeHash == NULL) { if (pData->dnodeHash == NULL) {
dError("failed to init dnode hash"); dError("failed to init dnode hash");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno = code;
} }
if (dmReadEps(pData) != 0) { if ((code = dmReadEps(pData)) != 0) {
dError("failed to read file since %s", terrstr()); dError("failed to read file since %s", tstrerror(code));
return -1; return code;
} }
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
@ -274,22 +287,21 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
} else { } else {
switch (pWrapper->ntype) { switch (pWrapper->ntype) {
case MNODE: case MNODE:
terrno = TSDB_CODE_MNODE_NOT_FOUND; code = TSDB_CODE_MNODE_NOT_FOUND;
break; break;
case QNODE: case QNODE:
terrno = TSDB_CODE_QNODE_NOT_FOUND; code = TSDB_CODE_QNODE_NOT_FOUND;
break; break;
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_NOT_FOUND; code = TSDB_CODE_SNODE_NOT_FOUND;
break; break;
case VNODE: case VNODE:
terrno = TSDB_CODE_VND_STOPPED; code = TSDB_CODE_VND_STOPPED;
break; break;
default: default:
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
break; break;
} }
code = -1;
} }
taosThreadRwlockUnlock(&pWrapper->lock); taosThreadRwlockUnlock(&pWrapper->lock);

47
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -17,12 +17,13 @@
#include "dmMgmt.h" #include "dmMgmt.h"
int32_t dmOpenNode(SMgmtWrapper *pWrapper) { int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
SDnode *pDnode = pWrapper->pDnode; SDnode *pDnode = pWrapper->pDnode;
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr()); dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code));
return -1; return code;
} }
SMgmtOutputOpt output = {0}; SMgmtOutputOpt output = {0};
@ -30,9 +31,9 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to open", pWrapper->name); dInfo("node:%s, start to open", pWrapper->name);
tmsgSetDefault(&input.msgCb); tmsgSetDefault(&input.msgCb);
if ((*pWrapper->func.openFp)(&input, &output) != 0) { if ((code = (*pWrapper->func.openFp)(&input, &output)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
dInfo("node:%s, has been opened", pWrapper->name); dInfo("node:%s, has been opened", pWrapper->name);
pWrapper->deployed = true; pWrapper->deployed = true;
@ -46,11 +47,12 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
} }
int32_t dmStartNode(SMgmtWrapper *pWrapper) { int32_t dmStartNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
if (pWrapper->func.startFp != NULL) { if (pWrapper->func.startFp != NULL) {
dDebug("node:%s, start to start", pWrapper->name); dDebug("node:%s, start to start", pWrapper->name);
if ((*pWrapper->func.startFp)(pWrapper->pMgmt) != 0) { if ((code = (*pWrapper->func.startFp)(pWrapper->pMgmt)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
dDebug("node:%s, has been started", pWrapper->name); dDebug("node:%s, has been started", pWrapper->name);
} }
@ -86,12 +88,13 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
} }
static int32_t dmOpenNodes(SDnode *pDnode) { static int32_t dmOpenNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (dmOpenNode(pWrapper) != 0) { if ((code = dmOpenNode(pWrapper)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
} }
@ -100,12 +103,13 @@ static int32_t dmOpenNodes(SDnode *pDnode) {
} }
static int32_t dmStartNodes(SDnode *pDnode) { static int32_t dmStartNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (dmStartNode(pWrapper) != 0) { if ((code = dmStartNode(pWrapper)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
} }
@ -129,19 +133,20 @@ static void dmCloseNodes(SDnode *pDnode) {
} }
int32_t dmRunDnode(SDnode *pDnode) { int32_t dmRunDnode(SDnode *pDnode) {
int32_t code = 0;
int32_t count = 0; int32_t count = 0;
if (dmOpenNodes(pDnode) != 0) { if ((code = dmOpenNodes(pDnode)) != 0) {
dError("failed to open nodes since %s", terrstr()); dError("failed to open nodes since %s", tstrerror(code));
dmCloseNodes(pDnode); dmCloseNodes(pDnode);
return -1; return code;
} }
if (dmStartNodes(pDnode) != 0) { if ((code = dmStartNodes(pDnode)) != 0) {
dError("failed to start nodes since %s", terrstr()); dError("failed to start nodes since %s", tstrerror(code));
dmSetStatus(pDnode, DND_STAT_STOPPED); dmSetStatus(pDnode, DND_STAT_STOPPED);
dmStopNodes(pDnode); dmStopNodes(pDnode);
dmCloseNodes(pDnode); dmCloseNodes(pDnode);
return -1; return code;
} }
while (1) { while (1) {

68
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -39,9 +39,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
if (msgFp == NULL) { if (msgFp == NULL) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED; // terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
return -1; return TSDB_CODE_MSG_NOT_PROCESSED;
} }
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
@ -54,14 +54,15 @@ static bool dmFailFastFp(tmsg_t msgType) {
return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES; return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES;
} }
static void dmConvertErrCode(tmsg_t msgType) { static int32_t dmConvertErrCode(tmsg_t msgType, int32_t code) {
if (terrno != TSDB_CODE_APP_IS_STOPPING) { if (code != TSDB_CODE_APP_IS_STOPPING) {
return; return code;
} }
if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) || if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) ||
(msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) { (msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) {
terrno = TSDB_CODE_VND_STOPPED; code = TSDB_CODE_VND_STOPPED;
} }
return code;
} }
static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) { static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite)); SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite));
@ -99,14 +100,14 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
int32_t svrVer = 0; int32_t svrVer = 0;
taosVersionStrToInt(version, &svrVer); taosVersionStrToInt(version, &svrVer);
if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) { if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) {
dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer); dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer);
goto _OVER; goto _OVER;
} }
bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp); bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp);
if (isForbidden) { if (isForbidden) {
terrno = TSDB_CODE_IP_NOT_IN_WHITE_LIST; code = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
goto _OVER; goto _OVER;
} }
@ -119,7 +120,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
case TDMT_SCH_FETCH_RSP: case TDMT_SCH_FETCH_RSP:
case TDMT_SCH_MERGE_FETCH_RSP: case TDMT_SCH_MERGE_FETCH_RSP:
case TDMT_VND_SUBMIT_RSP: case TDMT_VND_SUBMIT_RSP:
qWorkerProcessRspMsg(NULL, NULL, pRpc, 0); code = qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
return; return;
case TDMT_MND_STATUS_RSP: case TDMT_MND_STATUS_RSP:
if (pEpSet != NULL) { if (pEpSet != NULL) {
@ -148,32 +149,32 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
return; return;
} else { } else {
if (pDnode->status == DND_STAT_INIT) { if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
} else { } else {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
} }
goto _OVER; goto _OVER;
} }
} }
} else { } else {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
goto _OVER; goto _OVER;
} }
if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) { if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {
dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType)); dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN; code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER; goto _OVER;
} else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && } else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) &&
(!IsReq(pRpc)) && (pRpc->pCont == NULL)) { (!IsReq(pRpc)) && (pRpc->pCont == NULL)) {
dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code));
terrno = pRpc->code; code = pRpc->code;
goto _OVER; goto _OVER;
} }
if (pHandle->defaultNtype == NODE_END) { if (pHandle->defaultNtype == NODE_END) {
dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType)); dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
goto _OVER; goto _OVER;
} }
@ -197,12 +198,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
} }
} else { } else {
dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType)); dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN; code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER; goto _OVER;
} }
} }
if (dmMarkWrapper(pWrapper) != 0) { if ((code = dmMarkWrapper(pWrapper)) != 0) {
pWrapper = NULL; pWrapper = NULL;
goto _OVER; goto _OVER;
} }
@ -221,12 +222,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
dmConvertErrCode(pRpc->msgType); code = dmConvertErrCode(pRpc->msgType, code);
if (terrno != 0) code = terrno;
if (pMsg) { if (pMsg) {
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code));
} else { } else {
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr()); dGTrace("msg:%p, failed to process empty msg since %s", pMsg, tstrerror(code));
} }
if (IsReq(pRpc)) { if (IsReq(pRpc)) {
@ -280,17 +280,19 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
} }
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) { if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) { if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
} else { } else {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
} }
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
return -1; pMsg->info.handle);
return code;
} else { } else {
pMsg->info.handle = 0; pMsg->info.handle = 0;
rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL);
@ -298,17 +300,19 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
} }
} }
static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) { if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) { if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
} else { } else {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
} }
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
return -1; pMsg->info.handle);
return code;
} else { } else {
rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL); rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL);
return 0; return 0;
@ -426,7 +430,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
pTrans->statusRpc = rpcOpen(&rpcInit); pTrans->statusRpc = rpcOpen(&rpcInit);
if (pTrans->statusRpc == NULL) { if (pTrans->statusRpc == NULL) {
dError("failed to init dnode rpc status client"); dError("failed to init dnode rpc status client");
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
dDebug("dnode rpc status client is initialized"); dDebug("dnode rpc status client is initialized");
@ -471,7 +475,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
pTrans->syncRpc = rpcOpen(&rpcInit); pTrans->syncRpc = rpcOpen(&rpcInit);
if (pTrans->syncRpc == NULL) { if (pTrans->syncRpc == NULL) {
dError("failed to init dnode rpc sync client"); dError("failed to init dnode rpc sync client");
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
dDebug("dnode rpc sync client is initialized"); dDebug("dnode rpc sync client is initialized");
@ -521,7 +525,7 @@ int32_t dmInitServer(SDnode *pDnode) {
pTrans->serverRpc = rpcOpen(&rpcInit); pTrans->serverRpc = rpcOpen(&rpcInit);
if (pTrans->serverRpc == NULL) { if (pTrans->serverRpc == NULL) {
dError("failed to init dnode rpc server"); dError("failed to init dnode rpc server");
return -1; return terrno;
} }
dDebug("dnode rpc server is initialized"); dDebug("dnode rpc server is initialized");

3
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -205,7 +205,8 @@ void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
// dmFile.c // dmFile.c
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed); int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed); int32_t dmWriteFile(const char *path, const char *name, bool deployed);
TdFilePtr dmCheckRunning(const char *dataDir); int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c // dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData); int32_t dmInitDndInfo(SDnodeData *pData);

106
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -104,9 +104,9 @@ int dmOccurrences(char *str, char *toSearch) {
return count; return count;
} }
void dmSplitStr(char** arr, char* str, const char* del) { void dmSplitStr(char **arr, char *str, const char *del) {
char *lasts; char *lasts;
char* s = strsep(&str, del); char *s = strsep(&str, del);
while (s != NULL) { while (s != NULL) {
*arr++ = s; *arr++ = s;
s = strsep(&str, del); s = strsep(&str, del);
@ -132,11 +132,10 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("dnode file:%s not exist", file); dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){ if (strlen(tsEncryptAlgorithm) > 0) {
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) { if (strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4; pData->encryptAlgorigthm = DND_CA_SM4;
} } else {
else{
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm); dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER; goto _OVER;
@ -144,45 +143,45 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("start to parse encryptScope:%s", tsEncryptScope); dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope); int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){ if (scopeLen == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope); dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER; goto _OVER;
} }
char* tmp = taosMemoryMalloc(scopeLen + 1); char *tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1); memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen); memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ","); int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1)); char **array = taosMemoryMalloc(sizeof(char *) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1)); memset(array, 0, sizeof(char *) * (count + 1));
dmSplitStr(array, tmp, ","); dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){ for (int32_t i = 0; i < count + 1; i++) {
char* str = *(array + i); char *str = *(array + i);
bool success = false; bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_TSDB; pData->encryptScope |= DND_CS_TSDB;
success = true; success = true;
} }
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_VNODE_WAL; pData->encryptScope |= DND_CS_VNODE_WAL;
success = true; success = true;
} }
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_SDB; pData->encryptScope |= DND_CS_SDB;
success = true; success = true;
} }
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_MNODE_WAL; pData->encryptScope |= DND_CS_MNODE_WAL;
success = true; success = true;
} }
if(!success){ if (!success) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp); taosMemoryFree(tmp);
taosMemoryFree(array); taosMemoryFree(array);
@ -204,26 +203,26 @@ int32_t dmReadEps(SDnodeData *pData) {
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr()); dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr()); dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
@ -232,12 +231,11 @@ int32_t dmReadEps(SDnodeData *pData) {
pJson = tjsonParse(content); pJson = tjsonParse(content);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (dmDecodeEps(pJson, pData) < 0) { if ((code = dmDecodeEps(pJson, pData)) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -251,7 +249,7 @@ _OVER:
if (code != 0) { if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, terrstr());
return code; return terrno = code;
} }
if (taosArrayGetSize(pData->dnodeEps) == 0) { if (taosArrayGetSize(pData->dnodeEps) == 0) {
@ -261,8 +259,8 @@ _OVER:
taosArrayPush(pData->dnodeEps, &dnodeEp); taosArrayPush(pData->dnodeEps, &dnodeEp);
} }
if (dmReadDnodePairs(pData) != 0) { if ((code = dmReadDnodePairs(pData)) != 0) {
return -1; return terrno = code;
} }
dDebug("reset dnode list on startup"); dDebug("reset dnode list on startup");
@ -270,8 +268,8 @@ _OVER:
if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) { if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) {
dError("localEp %s different with %s and need to be reconfigured", tsLocalEp, file); dError("localEp %s different with %s and need to be reconfigured", tsLocalEp, file);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; return terrno = code;
} }
return code; return code;
@ -308,7 +306,7 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
} }
int32_t dmWriteEps(SDnodeData *pData) { int32_t dmWriteEps(SDnodeData *pData) {
int32_t code = -1; int32_t code = 0;
char *buffer = NULL; char *buffer = NULL;
SJson *pJson = NULL; SJson *pJson = NULL;
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
@ -317,26 +315,30 @@ int32_t dmWriteEps(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY; // if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmInitDndInfo(pData), NULL, _OVER);
if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
pData->engineVer = tsVersion; pData->engineVer = tsVersion;
if (dmEncodeEps(pJson, pData) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmEncodeEps(pJson, pData), NULL, _OVER); // dmEncodeEps(pJson, pData) != 0) goto _OVER;
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
if (taosFsyncFile(pFile) < 0) goto _OVER; if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
taosCloseFile(&pFile); taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER; if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
code = 0; code = 0;
pData->updateTime = taosGetTimestampMs(); pData->updateTime = taosGetTimestampMs();
@ -349,8 +351,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, tstrerror(code), pData->dnodeVer);
dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer);
} }
return code; return code;
} }
@ -592,28 +593,29 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file); code = TAOS_SYSTEM_ERROR(errno);
dDebug("dnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0; code = 0;
goto _OVER; goto _OVER;
} }
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr()); dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr()); dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -627,12 +629,13 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
pJson = tjsonParse(content); pJson = tjsonParse(content);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair)); pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair));
if (pData->oldDnodeEps == NULL) { if (pData->oldDnodeEps == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to calloc dnodeEp array since %s", strerror(errno)); dError("failed to calloc dnodeEp array since %s", strerror(errno));
goto _OVER; goto _OVER;
} }
@ -640,7 +643,8 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
if (dmDecodeEpPairs(pJson, pData) < 0) { if (dmDecodeEpPairs(pJson, pData) < 0) {
taosArrayDestroy(pData->oldDnodeEps); taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL; pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -653,7 +657,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, tstrerror(code));
return code; return code;
} }
@ -680,8 +684,8 @@ _OVER:
pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id); pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id);
taosArrayDestroy(pData->oldDnodeEps); taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL; pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; return code;
} }
} }
} }

320
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -14,11 +14,11 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmUtil.h"
#include "tjson.h"
#include "tgrant.h"
#include "crypt.h" #include "crypt.h"
#include "dmUtil.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "tgrant.h"
#include "tjson.h"
#define MAXLEN 1024 #define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!" #define DM_KEY_INDICATOR "this indicator!"
@ -30,7 +30,7 @@ static int32_t dmDecodeFile(SJson *pJson, bool *deployed) {
int32_t value = 0; int32_t value = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code); tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code);
if (code < 0) return -1; if (code < 0) return code;
*deployed = (value != 0); *deployed = (value != 0);
return code; return code;
@ -42,7 +42,11 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char *content = NULL; char *content = NULL;
SJson *pJson = NULL; SJson *pJson = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file); dInfo("file:%s not exist", file);
@ -52,27 +56,27 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", file, terrstr()); dError("failed to open file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat file:%s since %s", file, terrstr()); dError("failed to fstat file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read file:%s since %s", file, terrstr()); dError("failed to read file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -80,12 +84,12 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pJson = tjsonParse(content); pJson = tjsonParse(content);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (dmDecodeFile(pJson, pDeployed) < 0) { if (dmDecodeFile(pJson, pDeployed) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -98,13 +102,13 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, tstrerror(code));
} }
return code; return code;
} }
static int32_t dmEncodeFile(SJson *pJson, bool deployed) { static int32_t dmEncodeFile(SJson *pJson, bool deployed) {
if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
return 0; return 0;
} }
@ -115,26 +119,57 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) {
if (dmEncodeFile(pJson, deployed) != 0) goto _OVER; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = dmEncodeFile(pJson, deployed)) != 0) goto _OVER;
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosCloseFile(&pFile) != 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0; code = 0;
dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed); dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed);
@ -145,54 +180,58 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write file:%s since %s, deloyed:%d", realfile, tstrerror(code), deployed);
dError("failed to write file:%s since %s, deloyed:%d", realfile, terrstr(), deployed);
} }
return code; return code;
} }
TdFilePtr dmCheckRunning(const char *dataDir) { int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) {
int32_t code = 0;
char filepath[PATH_MAX] = {0}; char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP); snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC); *pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
if (pFile == NULL) { if (*pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", filepath, terrstr()); dError("failed to open file:%s since %s", filepath, tstrerror(code));
return NULL; return code;
} }
int32_t retryTimes = 0; int32_t retryTimes = 0;
int32_t ret = 0; int32_t ret = 0;
do { do {
ret = taosLockFile(pFile); ret = taosLockFile(*pFile);
if (ret == 0) break; if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
taosMsleep(1000); taosMsleep(1000);
retryTimes++; retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); dError("failed to lock file:%s since %s, retryTimes:%d", filepath, tstrerror(code), retryTimes);
} while (retryTimes < 12); } while (retryTimes < 12);
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(pFile);
return NULL; *pFile = NULL;
return code;
} }
terrno = 0;
dDebug("lock file:%s to prevent repeated starts", filepath); dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile; return code;
} }
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode); extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool toLogFile){ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool toLogFile) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char *result = NULL; char *result = NULL;
int32_t code = -1; int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR)); int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len); result = taosMemoryMalloc(len);
if (result == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SCryptOpts opts; SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN); strncpy(opts.key, key, ENCRYPT_KEY_LEN);
@ -203,48 +242,81 @@ static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool
CBC_Encrypt(&opts); CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER; if (taosWriteFile(pFile, opts.result, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosFsyncFile(pFile) < 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write checkCode file:%s", realfile); encryptDebug("succeed to write checkCode file:%s", realfile);
code = 0; code = 0;
_OVER: _OVER:
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result); if (result != NULL) taosMemoryFree(result);
return code; return code;
} }
static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode, bool toLogFile){ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptCode, bool toLogFile) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
int32_t code = -1; int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(encryptCode); int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER; if (taosWriteFile(pFile, encryptCode, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosCloseFile(&pFile) != 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write encryptCode file:%s", realfile); encryptDebug("succeed to write encryptCode file:%s", realfile);
code = 0; code = 0;
_OVER: _OVER:
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
return code; return code;
} }
static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) {
char *content = NULL; char *content = NULL;
int64_t size = 0; int64_t size = 0;
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
@ -253,26 +325,26 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to open dnode file:%s since %s", file, terrstr()); encryptError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to fstat dnode file:%s since %s", file, terrstr()); encryptError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size); content = taosMemoryMalloc(size);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to read dnode file:%s since %s", file, terrstr()); encryptError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -289,8 +361,8 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
opts.unitLen = 16; opts.unitLen = 16;
CBC_Decrypt(&opts); CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) { if (strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED; code = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
encryptError("failed to compare decrypted result"); encryptError("failed to compare decrypted result");
goto _OVER; goto _OVER;
} }
@ -298,9 +370,9 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
encryptDebug("succeed to compare checkCode file:%s", file); encryptDebug("succeed to compare checkCode file:%s", file);
code = 0; code = 0;
_OVER: _OVER:
if(result != NULL) taosMemoryFree(result); if (result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content); if (content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
return code; return code;
} }
@ -320,38 +392,55 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
char checkFile[PATH_MAX] = {0}; char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0}; char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP); int32_t nBytes = snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); if (nBytes <= 0 || nBytes >= PATH_MAX) {
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); return TSDB_CODE_OUT_OF_BUFFER;
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE); }
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
terrno = 0; nBytes = snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
if (taosMkDir(folder) != 0) { if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to create dir:%s since %s", folder, terrstr()); encryptError("failed to create dir:%s since %s", folder, tstrerror(code));
goto _OVER; goto _OVER;
} }
if(taosCheckExistFile(realCheckFile)){ if (taosCheckExistFile(realCheckFile)) {
if(dmCompareEncryptKey(realCheckFile, key, toLogFile) != 0){ if ((code = dmCompareEncryptKey(realCheckFile, key, toLogFile)) != 0) {
goto _OVER; goto _OVER;
} }
} }
if (!(machineId = tGetMachineId())) { if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
TAOS_CHECK_GOTO(generateEncryptCode(key, machineId, &encryptCode), &lino, _OVER); TAOS_CHECK_GOTO(generateEncryptCode(key, machineId, &encryptCode), &lino, _OVER);
if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){ if ((code = dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile)) != 0) {
goto _OVER; goto _OVER;
} }
if(dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile) != 0){ if ((code = dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile)) != 0) {
goto _OVER; goto _OVER;
} }
@ -362,10 +451,9 @@ _OVER:
taosMemoryFree(encryptCode); taosMemoryFree(encryptCode);
taosMemoryFree(machineId); taosMemoryFree(machineId);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); encryptError("failed to update encrypt key at line %d since %s", lino, tstrerror(code));
encryptError("failed to update encrypt key since %s", terrstr());
} }
return code; TAOS_RETURN(code);
#else #else
return 0; return 0;
#endif #endif
@ -373,50 +461,52 @@ _OVER:
extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key); extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
static int32_t dmReadEncryptCodeFile(char* file, char** output){ static int32_t dmReadEncryptCodeFile(char *file, char **output) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
int32_t code = -1; int32_t code = -1;
char *content = NULL; char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr()); dError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr()); dError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content[size] = '\0'; content[size] = '\0';
*output = content; *output = content;
content = NULL;
dInfo("succeed to read encryptCode file:%s", file); dInfo("succeed to read encryptCode file:%s", file);
code = 0; code = 0;
_OVER: _OVER:
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
taosMemoryFree(content);
return code; return code;
} }
int32_t dmGetEncryptKey(){ int32_t dmGetEncryptKey() {
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
int32_t code = -1; int32_t code = -1;
char encryptFile[PATH_MAX] = {0}; char encryptFile[PATH_MAX] = {0};
@ -425,37 +515,48 @@ int32_t dmGetEncryptKey(){
char *encryptKey = NULL; char *encryptKey = NULL;
char *content = NULL; char *content = NULL;
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); int32_t nBytes = snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP,
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE); DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(encryptFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if(!taosCheckExistFile(encryptFile)){ nBytes = snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(checkFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if (!taosCheckExistFile(encryptFile)) {
code = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dInfo("no exist, checkCode file:%s", encryptFile); dInfo("no exist, checkCode file:%s", encryptFile);
return 0; return 0;
} }
if(dmReadEncryptCodeFile(encryptFile, &content) != 0){ if ((code = dmReadEncryptCodeFile(encryptFile, &content)) != 0) {
goto _OVER; goto _OVER;
} }
if (!(machineId = tGetMachineId())) { if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if(checkAndGetCryptKey(content, machineId, &encryptKey) != 0){ if ((code = checkAndGetCryptKey(content, machineId, &encryptKey)) != 0) {
goto _OVER; goto _OVER;
} }
taosMemoryFreeClear(machineId); taosMemoryFreeClear(machineId);
taosMemoryFreeClear(content); taosMemoryFreeClear(content);
if(encryptKey[0] == '\0'){ if (encryptKey[0] == '\0') {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr()); dError("failed to read key since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
if(dmCompareEncryptKey(checkFile, encryptKey, true) != 0){ if ((code = dmCompareEncryptKey(checkFile, encryptKey, true)) != 0) {
goto _OVER; goto _OVER;
} }
@ -470,8 +571,7 @@ _OVER:
if (encryptKey != NULL) taosMemoryFree(encryptKey); if (encryptKey != NULL) taosMemoryFree(encryptKey);
if (machineId != NULL) taosMemoryFree(machineId); if (machineId != NULL) taosMemoryFree(machineId);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to get encrypt key since %s", tstrerror(code));
dError("failed to get encrypt key since %s", terrstr());
} }
return code; return code;
#else #else

60
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -54,6 +54,7 @@ int32_t mndInitAcct(SMnode *pMnode) {
void mndCleanupAcct(SMnode *pMnode) {} void mndCleanupAcct(SMnode *pMnode) {}
static int32_t mndCreateDefaultAcct(SMnode *pMnode) { static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
int32_t code = 0;
SAcctObj acctObj = {0}; SAcctObj acctObj = {0};
tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN);
acctObj.createdTime = taosGetTimestampMs(); acctObj.createdTime = taosGetTimestampMs();
@ -76,7 +77,10 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
}; };
SSdbRaw *pRaw = mndAcctActionEncode(&acctObj); SSdbRaw *pRaw = mndAcctActionEncode(&acctObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw); mInfo("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw);
@ -84,21 +88,24 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct");
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
mError("acct:%s, failed to create since %s", acctObj.acct, terrstr()); code = terrno;
return -1; mError("acct:%s, failed to create since %s", acctObj.acct, tstrerror(code));
TAOS_RETURN(code);
} }
mInfo("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct); mInfo("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { code = mndTransAppendCommitlog(pTrans, pRaw);
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); if (code != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, tstrerror(code));
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); if (code != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code));
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -216,31 +223,36 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
} }
static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { int32_t code = 0;
return -1; code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT);
if (code != 0) {
TAOS_RETURN(code);
} }
terrno = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { int32_t code = 0;
return -1; code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT);
if (code != 0) {
TAOS_RETURN(code);
} }
terrno = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { int32_t code = 0;
return -1; if ((code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT)) != 0) {
TAOS_RETURN(code);
} }
terrno = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }

197
source/dnode/mnode/impl/src/mndArbGroup.c
View File

@ -57,6 +57,7 @@ static int32_t mndArbCheckToken(const char *token1, const char *token2) {
} }
int32_t mndInitArbGroup(SMnode *pMnode) { int32_t mndInitArbGroup(SMnode *pMnode) {
int32_t code = 0;
SSdbTable table = { SSdbTable table = {
.sdbType = SDB_ARBGROUP, .sdbType = SDB_ARBGROUP,
.keyType = SDB_KEY_INT32, .keyType = SDB_KEY_INT32,
@ -78,6 +79,10 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup);
arbUpdateHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); arbUpdateHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
if (arbUpdateHash == NULL) {
code = terrno;
TAOS_RETURN(code);
}
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
@ -248,50 +253,74 @@ _OVER:
} }
int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendRedolog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pUndoRaw = mndArbGroupActionEncode(pGroup);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendUndolog(pTrans, pUndoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetCreateArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetCreateArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw) != 0)) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetDropArbGroupPrepareLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetDropArbGroupPrepareLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
static int32_t mndSetDropArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) { static int32_t mndSetDropArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendRedolog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetDropArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetDropArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
@ -343,6 +372,7 @@ static int32_t mndSendArbHeartBeatReq(SDnodeObj *pDnode, char *arbToken, int64_t
} }
static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) { static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL; SArbGroup *pArbGroup = NULL;
@ -377,7 +407,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
} }
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-hb timer"); mError("failed to get arb token for arb-hb timer");
pIter = taosHashIterate(pDnodeHash, NULL); pIter = taosHashIterate(pDnodeHash, NULL);
while (pIter) { while (pIter) {
@ -386,7 +416,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
pIter = taosHashIterate(pDnodeHash, pIter); pIter = taosHashIterate(pDnodeHash, pIter);
} }
taosHashCleanup(pDnodeHash); taosHashCleanup(pDnodeHash);
return -1; TAOS_RETURN(code);
} }
int64_t nowMs = taosGetTimestampMs(); int64_t nowMs = taosGetTimestampMs();
@ -447,6 +477,7 @@ static void *mndBuildArbCheckSyncReq(int32_t *pContLen, int32_t vgId, char *arbT
static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbToken, int64_t term, char *member0Token, static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbToken, int64_t term, char *member0Token,
char *member1Token) { char *member1Token) {
int32_t code = 0;
int32_t contLen = 0; int32_t contLen = 0;
void *pHead = mndBuildArbCheckSyncReq(&contLen, vgId, arbToken, term, member0Token, member1Token); void *pHead = mndBuildArbCheckSyncReq(&contLen, vgId, arbToken, term, member0Token, member1Token);
if (!pHead) { if (!pHead) {
@ -459,10 +490,12 @@ static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbTok
if (epSet.numOfEps == 0) { if (epSet.numOfEps == 0) {
mError("vgId:%d, failed to send check-sync request since no epSet found", vgId); mError("vgId:%d, failed to send check-sync request since no epSet found", vgId);
rpcFreeCont(pHead); rpcFreeCont(pHead);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int32_t code = tmsgSendReq(&epSet, &rpcMsg); code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
mError("vgId:%d, failed to send check-sync request since 0x%x", vgId, code); mError("vgId:%d, failed to send check-sync request since 0x%x", vgId, code);
} else { } else {
@ -502,11 +535,14 @@ static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, ch
static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken, static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken,
int64_t term, char *memberToken) { int64_t term, char *memberToken) {
int32_t code = 0;
int32_t contLen = 0; int32_t contLen = 0;
void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken); void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken);
if (!pHead) { if (!pHead) {
mError("vgId:%d, failed to build set-assigned request", vgId); mError("vgId:%d, failed to build set-assigned request", vgId);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
SRpcMsg rpcMsg = {.msgType = TDMT_SYNC_SET_ASSIGNED_LEADER, .pCont = pHead, .contLen = contLen}; SRpcMsg rpcMsg = {.msgType = TDMT_SYNC_SET_ASSIGNED_LEADER, .pCont = pHead, .contLen = contLen};
@ -514,9 +550,11 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
if (epSet.numOfEps == 0) { if (epSet.numOfEps == 0) {
mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since no epSet found", dnodeId, vgId); mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since no epSet found", dnodeId, vgId);
rpcFreeCont(pHead); rpcFreeCont(pHead);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int32_t code = tmsgSendReq(&epSet, &rpcMsg); code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since 0x%x", dnodeId, vgId, code); mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since 0x%x", dnodeId, vgId, code);
} else { } else {
@ -526,6 +564,7 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
} }
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL; SArbGroup *pArbGroup = NULL;
@ -533,14 +572,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
void *pIter = NULL; void *pIter = NULL;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-check-sync timer"); mError("failed to get arb token for arb-check-sync timer");
return -1; TAOS_RETURN(code);
} }
int64_t term = mndGetTerm(pMnode); int64_t term = mndGetTerm(pMnode);
if (term < 0) { if (term < 0) {
mError("arb failed to get term since %s", terrstr()); mError("arb failed to get term since %s", terrstr());
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int64_t roleTimeMs = mndGetRoleTimeMs(pMnode); int64_t roleTimeMs = mndGetRoleTimeMs(pMnode);
@ -747,13 +788,13 @@ _OVER:
} }
static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) { static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
int ret = -1; int code = -1;
size_t sz = 0; size_t sz = 0;
SMArbUpdateGroupBatchReq req = {0}; SMArbUpdateGroupBatchReq req = {0};
if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) { if ((code = tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req)) != 0) {
mError("arb failed to decode arb-update-group request"); mError("arb failed to decode arb-update-group request");
return -1; TAOS_RETURN(code);
} }
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -789,7 +830,7 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
mndTransAddArbGroupId(pTrans, newGroup.vgId); mndTransAddArbGroupId(pTrans, newGroup.vgId);
if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) { if ((code = mndSetCreateArbGroupCommitLogs(pTrans, &newGroup)) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id, mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id,
terrstr()); terrstr());
goto _OVER; goto _OVER;
@ -803,13 +844,13 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pOldGroup); sdbRelease(pMnode->pSdb, pOldGroup);
} }
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
ret = 0; code = 0;
_OVER: _OVER:
if (ret != 0) { if (code != 0) {
// failed to update arbgroup // failed to update arbgroup
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i); SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
@ -819,7 +860,7 @@ _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
tFreeSMArbUpdateGroupBatchReq(&req); tFreeSMArbUpdateGroupBatchReq(&req);
return ret; return code;
} }
static void mndArbGroupDupObj(SArbGroup *pGroup, SArbGroup *pNew) { static void mndArbGroupDupObj(SArbGroup *pGroup, SArbGroup *pNew) {
@ -841,10 +882,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) {
} }
static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) { static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
int32_t ret = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to update arbgroup in create trans, vgId:%d, since %s", pNew->vgId, terrstr()); mError("failed to update arbgroup in create trans, vgId:%d, since %s", pNew->vgId, terrstr());
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -854,23 +896,22 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
pNew->assignedLeader.token, pNew->assignedLeader.acked); pNew->assignedLeader.token, pNew->assignedLeader.acked);
mndTransAddArbGroupId(pTrans, pNew->vgId); mndTransAddArbGroupId(pTrans, pNew->vgId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) {
ret = -1;
goto _OVER; goto _OVER;
} }
if (mndSetCreateArbGroupCommitLogs(pTrans, pNew) != 0) { if ((code = mndSetCreateArbGroupCommitLogs(pTrans, pNew)) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, since %s", pNew->vgId, terrstr()); mError("failed to update arbgroup in set commit log, vgId:%d, since %s", pNew->vgId, tstrerror(code));
goto _OVER; goto _OVER;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
ret = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return ret; return code;
} }
bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMember, int64_t nowMs, int32_t dnodeId, bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMember, int64_t nowMs, int32_t dnodeId,
@ -993,11 +1034,13 @@ _OVER:
} }
static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token, char *member1Token, bool newIsSync) { static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token, char *member1Token, bool newIsSync) {
int32_t code = 0;
SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId); SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
if (pGroup == NULL) { if (pGroup == NULL) {
terrno = TSDB_CODE_NOT_FOUND;
mInfo("failed to update arb sync, vgId:%d not found", vgId); mInfo("failed to update arb sync, vgId:%d not found", vgId);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
@ -1018,38 +1061,36 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t ret = -1; int32_t code = -1;
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-hb response"); mError("failed to get arb token for arb-hb response");
terrno = TSDB_CODE_NOT_FOUND; TAOS_RETURN(code);
return -1;
} }
SVArbHeartBeatRsp arbHbRsp = {0}; SVArbHeartBeatRsp arbHbRsp = {0};
if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) { if ((code = tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp)) != 0) {
mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code)); mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(code);
return -1;
} }
if (mndArbCheckToken(arbToken, arbHbRsp.arbToken) != 0) { if (mndArbCheckToken(arbToken, arbHbRsp.arbToken) != 0) {
mInfo("arb hearbeat skip update for dnodeId:%d, arb token mismatch, local:[%s] msg:[%s]", arbHbRsp.dnodeId, mInfo("arb hearbeat skip update for dnodeId:%d, arb token mismatch, local:[%s] msg:[%s]", arbHbRsp.dnodeId,
arbToken, arbHbRsp.arbToken); arbToken, arbHbRsp.arbToken);
terrno = TSDB_CODE_MND_ARB_TOKEN_MISMATCH; code = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
goto _OVER; goto _OVER;
} }
(void)mndUpdateArbHeartBeat(pMnode, arbHbRsp.dnodeId, arbHbRsp.hbMembers); (void)mndUpdateArbHeartBeat(pMnode, arbHbRsp.dnodeId, arbHbRsp.hbMembers);
ret = 0; code = 0;
_OVER: _OVER:
tFreeSVArbHeartBeatRsp(&arbHbRsp); tFreeSVArbHeartBeatRsp(&arbHbRsp);
return ret; return code;
} }
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) { static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
@ -1058,27 +1099,25 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t ret = -1; int32_t code = -1;
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-check-sync response"); mError("failed to get arb token for arb-check-sync response");
terrno = TSDB_CODE_NOT_FOUND; TAOS_RETURN(code);
return -1;
} }
SVArbCheckSyncRsp syncRsp = {0}; SVArbCheckSyncRsp syncRsp = {0};
if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) { if ((code = tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp)) != 0) {
mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code)); mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code));
if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) { if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
return 0; return 0;
} }
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(code);
return -1;
} }
if (mndArbCheckToken(arbToken, syncRsp.arbToken) != 0) { if (mndArbCheckToken(arbToken, syncRsp.arbToken) != 0) {
@ -1089,16 +1128,16 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
} }
bool newIsSync = (syncRsp.errCode == TSDB_CODE_SUCCESS); bool newIsSync = (syncRsp.errCode == TSDB_CODE_SUCCESS);
if (mndUpdateArbSync(pMnode, syncRsp.vgId, syncRsp.member0Token, syncRsp.member1Token, newIsSync) != 0) { if ((code = mndUpdateArbSync(pMnode, syncRsp.vgId, syncRsp.member0Token, syncRsp.member1Token, newIsSync)) != 0) {
mInfo("failed to update arb sync for vgId:%d, since:%s", syncRsp.vgId, terrstr()); mInfo("failed to update arb sync for vgId:%d, since:%s", syncRsp.vgId, terrstr());
goto _OVER; goto _OVER;
} }
ret = 0; code = 0;
_OVER: _OVER:
tFreeSVArbCheckSyncRsp(&syncRsp); tFreeSVArbCheckSyncRsp(&syncRsp);
return ret; TAOS_RETURN(code);
} }
bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char *memberToken, int32_t errcode, bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char *memberToken, int32_t errcode,
@ -1138,35 +1177,35 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t ret = -1; int32_t code = -1;
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-set-assigned response"); mError("failed to get arb token for arb-set-assigned response");
terrno = TSDB_CODE_NOT_FOUND; TAOS_RETURN(code);
return -1;
} }
SVArbSetAssignedLeaderRsp setAssignedRsp = {0}; SVArbSetAssignedLeaderRsp setAssignedRsp = {0};
if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) { if ((code = tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp)) != 0) {
mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code)); mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(code);
return -1;
} }
if (mndArbCheckToken(arbToken, setAssignedRsp.arbToken) != 0) { if (mndArbCheckToken(arbToken, setAssignedRsp.arbToken) != 0) {
mInfo("skip update arb assigned for vgId:%d, arb token mismatch, local:[%s] msg:[%s]", setAssignedRsp.vgId, mInfo("skip update arb assigned for vgId:%d, arb token mismatch, local:[%s] msg:[%s]", setAssignedRsp.vgId,
arbToken, setAssignedRsp.arbToken); arbToken, setAssignedRsp.arbToken);
terrno = TSDB_CODE_MND_ARB_TOKEN_MISMATCH; code = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
goto _OVER; goto _OVER;
} }
SArbGroup *pGroup = mndAcquireArbGroup(pMnode, setAssignedRsp.vgId); SArbGroup *pGroup = mndAcquireArbGroup(pMnode, setAssignedRsp.vgId);
if (!pGroup) { if (!pGroup) {
mError("failed to set arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr()); mError("failed to set arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr());
code = -1;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -1174,17 +1213,17 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(pGroup, setAssignedRsp.vgId, setAssignedRsp.memberToken, bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(pGroup, setAssignedRsp.vgId, setAssignedRsp.memberToken,
pRsp->code, &newGroup); pRsp->code, &newGroup);
if (updateAssigned) { if (updateAssigned) {
if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) { if ((code = mndPullupArbUpdateGroup(pMnode, &newGroup)) != 0) {
mInfo("failed to pullup update arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr()); mInfo("failed to pullup update arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, tstrerror(code));
goto _OVER; goto _OVER;
} }
} }
ret = 0; code = 0;
_OVER: _OVER:
tFreeSVArbSetAssignedLeaderRsp(&setAssignedRsp); tFreeSVArbSetAssignedLeaderRsp(&setAssignedRsp);
return ret; return code;
} }
static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

56
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -14,8 +14,8 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "audit.h"
#include "mndCluster.h" #include "mndCluster.h"
#include "audit.h"
#include "mndGrant.h" #include "mndGrant.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
#include "mndShow.h" #include "mndShow.h"
@ -61,11 +61,13 @@ int32_t mndInitCluster(SMnode *pMnode) {
void mndCleanupCluster(SMnode *pMnode) {} void mndCleanupCluster(SMnode *pMnode) {}
int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId);
if (pCluster == NULL) { if (pCluster == NULL) {
return -1; code = terrno;
TAOS_RETURN(code);
} }
tstrncpy(clusterName, pCluster->name, len); tstrncpy(clusterName, pCluster->name, len);
@ -247,7 +249,10 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
mInfo("cluster:%" PRId64 ", name is %s", clusterObj.id, clusterObj.name); mInfo("cluster:%" PRId64 ", name is %s", clusterObj.id, clusterObj.name);
SSdbRaw *pRaw = mndClusterActionEncode(&clusterObj); SSdbRaw *pRaw = mndClusterActionEncode(&clusterObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw); mInfo("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw);
@ -256,21 +261,22 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr());
return -1; code = terrno;
TAOS_RETURN(code);
} }
mInfo("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id); mInfo("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -354,22 +360,33 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
return 0; return 0;
} }
int32_t code = 0;
mInfo("update cluster uptime to %d", clusterObj.upTime); mInfo("update cluster uptime to %d", clusterObj.upTime);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime");
if (pTrans == NULL) return -1; if (pTrans == NULL) {
code = terrno;
TAOS_RETURN(code);
}
SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -381,13 +398,12 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SMCfgClusterReq cfgReq = {0}; SMCfgClusterReq cfgReq = {0};
if (tDeserializeSMCfgClusterReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { if (tDeserializeSMCfgClusterReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; TAOS_RETURN(code);
} }
mInfo("cluster: start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); mInfo("cluster: start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER) != 0) { if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER)) != 0) {
code = terrno != 0 ? terrno : TSDB_CODE_MND_NO_RIGHTS;
goto _exit; goto _exit;
} }
@ -417,17 +433,17 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
} }
{ // audit { // audit
auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql,
TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1));
} }
_exit: _exit:
tFreeSMCfgClusterReq(&cfgReq); tFreeSMCfgClusterReq(&cfgReq);
if (code != 0) { if (code != 0) {
terrno = code;
mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr()); mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr());
} else { } else {
mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value); mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value);
} }
return code; TAOS_RETURN(code);
} }
int32_t mndProcessConfigClusterRsp(SRpcMsg *pRsp) { int32_t mndProcessConfigClusterRsp(SRpcMsg *pRsp) {

183
source/dnode/mnode/impl/src/mndCompact.c
View File

@ -75,11 +75,11 @@ int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen); tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(&decoder));
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactId));
if (tDecodeCStrTo(&decoder, pObj->dbname) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(&decoder, pObj->dbname));
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(&decoder, &pObj->startTime));
tEndDecode(&decoder); tEndDecode(&decoder);
@ -174,13 +174,10 @@ SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
} }
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER); SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactObj(buf, tlen, pCompact) < 0) { if ((terrno = tDeserializeSCompactObj(buf, tlen, pCompact)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER; goto OVER;
} }
// taosInitRWLatch(&pView->lock);
OVER: OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
@ -227,6 +224,7 @@ void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
// compact db // compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) { int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) {
int32_t code = 0;
pCompact->compactId = tGenIdPI32(); pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name); strcpy(pCompact->dbname, pDb->name);
@ -234,10 +232,14 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompac
pCompact->startTime = taosGetTimestampMs(); pCompact->startTime = taosGetTimestampMs();
SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact); SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pVgRaw) != 0) { code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendPrepareLog(pTrans, pVgRaw)) != 0) {
sdbFreeRaw(pVgRaw); sdbFreeRaw(pVgRaw);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
@ -335,44 +337,62 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId, static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId,
int32_t dnodeid) { int32_t dnodeid) {
int32_t code = 0;
STransAction action = {0}; STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid); SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
if (pDnode == NULL) return -1; if (pDnode == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
action.epSet = mndGetDnodeEpset(pDnode); action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid); void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid);
if (pReq == NULL) return -1; if (pReq == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
action.pCont = pReq; action.pCont = pReq;
action.contLen = contLen; action.contLen = contLen;
action.msgType = TDMT_VND_KILL_COMPACT; action.msgType = TDMT_VND_KILL_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; return 0;
} }
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) { static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr()); mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId); mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
mndTransSetDbName(pTrans, pCompact->dbname, NULL); mndTransSetDbName(pTrans, pCompact->dbname, NULL);
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
@ -386,14 +406,20 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId); SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if (pVgroup == NULL) { if (pVgroup == NULL) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
if (mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0) { if ((code = mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId)) != 0) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndReleaseVgroup(pMnode, pVgroup); mndReleaseVgroup(pMnode, pVgroup);
@ -412,10 +438,10 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -423,30 +449,27 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
} }
int32_t mndProcessKillCompactReq(SRpcMsg *pReq) { int32_t mndProcessKillCompactReq(SRpcMsg *pReq) {
int32_t code = 0;
int32_t lino = 0;
SKillCompactReq killCompactReq = {0}; SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; if ((code = tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq)) != 0) {
return -1; TAOS_RETURN(code);
} }
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId); mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId); SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if (pCompact == NULL) { if (pCompact == NULL) {
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID; code = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq); tFreeSKillCompactReq(&killCompactReq);
return -1; TAOS_RETURN(code);
} }
if (0 != mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB)) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB), &lino, _OVER);
goto _OVER;
}
if (mndKillCompact(pMnode, pReq, pCompact) < 0) { TAOS_CHECK_GOTO(mndKillCompact(pMnode, pReq, pCompact), &lino, _OVER);
goto _OVER;
}
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
@ -463,12 +486,14 @@ _OVER:
tFreeSKillCompactReq(&killCompactReq); tFreeSKillCompactReq(&killCompactReq);
sdbRelease(pMnode->pSdb, pCompact); sdbRelease(pMnode->pSdb, pCompact);
return code; TAOS_RETURN(code);
} }
// update progress // update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId,
SQueryCompactProgressRsp *rsp) { SQueryCompactProgressRsp *rsp) {
int32_t code = 0;
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
SCompactDetailObj *pDetail = NULL; SCompactDetailObj *pDetail = NULL;
@ -479,9 +504,10 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
pDetail->newNumberFileset = rsp->numberFileset; pDetail->newNumberFileset = rsp->numberFileset;
pDetail->newFinished = rsp->finished; pDetail->newFinished = rsp->finished;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
return 0; TAOS_RETURN(code);
} }
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
@ -491,14 +517,13 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
} }
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) { int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
SQueryCompactProgressRsp req = {0};
int32_t code = 0; int32_t code = 0;
SQueryCompactProgressRsp req = {0};
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req); code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_INVALID_MSG;
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont, mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont,
pReq->contLen); pReq->contLen);
return -1; TAOS_RETURN(code);
} }
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
@ -508,13 +533,12 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req); code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
if (code != 0) { if (code != 0) {
terrno = code;
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished); req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
// timer // timer
@ -531,7 +555,10 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if (pDnode == NULL) break; if (pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port); if (addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port) != 0) {
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
SQueryCompactProgressReq req; SQueryCompactProgressReq req;
@ -541,8 +568,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req); int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) { if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
continue; continue;
} }
@ -551,7 +576,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SMsgHead *pHead = rpcMallocCont(contLen); SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) { if (pHead == NULL) {
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
continue; continue;
} }
@ -563,13 +587,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen}; SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen};
// rpcMsg.pCont = rpcMallocCont(contLen);
// if (rpcMsg.pCont == NULL) {
// return;
// }
// memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead; rpcMsg.pCont = pHead;
char detail[1024] = {0}; char detail[1024] = {0};
@ -589,6 +606,7 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
} }
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
int32_t code = 0;
bool needSave = false; bool needSave = false;
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
@ -612,7 +630,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
} }
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
if (pCompact == NULL) return 0; if (pCompact == NULL) TAOS_RETURN(code);
SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname); SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) { if (pDb == NULL) {
@ -625,13 +643,15 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (!needSave) { if (!needSave) {
mDebug("compact:%" PRId32 ", no need to save", compactId); mDebug("compact:%" PRId32 ", no need to save", compactId);
return 0; TAOS_RETURN(code);
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr()); mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id); mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
@ -654,10 +674,20 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
pDetail->finished = pDetail->newFinished; pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail); SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr()); sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
} }
@ -678,11 +708,13 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pDetail->numberFileset == -1 && pDetail->finished == -1) { if (pDetail->numberFileset == -1 && pDetail->finished == -1) {
allFinished = false; allFinished = false;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
break; break;
} }
if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) { if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) {
allFinished = false; allFinished = false;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
break; break;
} }
@ -710,11 +742,19 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pDetail->compactId == pCompact->compactId) { if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail); SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mndTransDrop(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id,
terrstr()); terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId); mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId);
@ -724,20 +764,26 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
} }
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mndTransDrop(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr()); mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compact action", pCompact->compactId); mInfo("compact:%d, add drop compact action", pCompact->compactId);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr()); mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
sdbRelease(pMnode->pSdb, pCompact); sdbRelease(pMnode->pSdb, pCompact);
return -1; TAOS_RETURN(code);
} }
sdbRelease(pMnode->pSdb, pCompact); sdbRelease(pMnode->pSdb, pCompact);
@ -746,6 +792,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
} }
void mndCompactPullup(SMnode *pMnode) { void mndCompactPullup(SMnode *pMnode) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t)); SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t));
if (pArray == NULL) return; if (pArray == NULL) return;
@ -766,7 +813,9 @@ void mndCompactPullup(SMnode *pMnode) {
if (pCompact != NULL) { if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId); mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact); mndCompactSendProgressReq(pMnode, pCompact);
mndSaveCompactProgress(pMnode, pCompact->compactId); if ((code = mndSaveCompactProgress(pMnode, pCompact->compactId)) != 0) {
mError("compact:%d, failed to save compact progress since %s", pCompact->compactId, tstrerror(code));
}
} }
mndReleaseCompact(pMnode, pCompact); mndReleaseCompact(pMnode, pCompact);
} }

32
source/dnode/mnode/impl/src/mndCompactDetail.c
View File

@ -125,17 +125,17 @@ int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailO
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen); tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(&decoder));
if (tDecodeI32(&decoder, &pObj->compactDetailId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactDetailId));
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactId));
if (tDecodeI32(&decoder, &pObj->vgId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->vgId));
if (tDecodeI32(&decoder, &pObj->dnodeId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->dnodeId));
if (tDecodeI32(&decoder, &pObj->numberFileset) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->numberFileset));
if (tDecodeI32(&decoder, &pObj->finished) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->finished));
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(&decoder, &pObj->startTime));
if (tDecodeI32(&decoder, &pObj->newNumberFileset) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->newNumberFileset));
if (tDecodeI32(&decoder, &pObj->newFinished) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->newFinished));
tEndDecode(&decoder); tEndDecode(&decoder);
@ -231,13 +231,10 @@ SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) {
} }
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER); SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactDetailObj(buf, tlen, pCompact) < 0) { if ((terrno = tDeserializeSCompactDetailObj(buf, tlen, pCompact)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER; goto OVER;
} }
//taosInitRWLatch(&pView->lock);
OVER: OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
@ -274,6 +271,7 @@ int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact,
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup, int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index){ SVnodeGid *pVgid, int32_t index){
int32_t code = 0;
SCompactDetailObj compactDetail = {0}; SCompactDetailObj compactDetail = {0};
compactDetail.compactDetailId = index; compactDetail.compactDetailId = index;
compactDetail.compactId = pCompact->compactId; compactDetail.compactId = pCompact->compactId;
@ -292,9 +290,11 @@ int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* p
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw); sdbFreeRaw(pVgRaw);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0; TAOS_RETURN(code);
} }

129
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -95,26 +95,21 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
bool enableReplay) { bool enableReplay) {
SMqTopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
int32_t numOfTopics = taosArrayGetSize(pTopicList); int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) { for (int32_t i = 0; i < numOfTopics; i++) {
char *pOneTopic = taosArrayGetP(pTopicList, i); char *pOneTopic = taosArrayGetP(pTopicList, i);
pTopic = mndAcquireTopic(pMnode, pOneTopic); pTopic = mndAcquireTopic(pMnode, pOneTopic);
if (pTopic == NULL) { // terrno has been set by callee function if (pTopic == NULL) { // terrno has been set by callee function
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED; goto FAILED;
} }
if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) { TAOS_CHECK_GOTO(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic), &lino, FAILED);
code = TSDB_CODE_MND_NO_RIGHTS;
terrno = TSDB_CODE_MND_NO_RIGHTS;
goto FAILED;
}
if ((terrno = grantCheckExpire(TSDB_GRANT_SUBSCRIPTION)) < 0) { TAOS_CHECK_GOTO(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION), &lino, FAILED);
code = terrno;
goto FAILED;
}
if (enableReplay) { if (enableReplay) {
if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
@ -123,7 +118,8 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
} else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { } else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
if (pDb == NULL) { if (pDb == NULL) {
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED; goto FAILED;
} }
if (pDb->cfg.numOfVgroups != 1) { if (pDb->cfg.numOfVgroups != 1) {
@ -141,11 +137,12 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
return 0; return 0;
FAILED: FAILED:
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont; SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
@ -153,7 +150,8 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId); mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId);
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
@ -161,42 +159,39 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
pConsumer->status, mndConsumerStatusName(pConsumer->status)); pConsumer->status, mndConsumerStatusName(pConsumer->status));
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) { if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; code = TSDB_CODE_MND_CONSUMER_NOT_READY;
code = -1;
goto END; goto END;
} }
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL); pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL);
if (pConsumerNew == NULL){ if (pConsumerNew == NULL){
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) { if (pTrans == NULL) {
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
goto END; if (terrno != 0) code = terrno;
}
code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
if (code != 0) {
goto END; goto END;
} }
code = mndSetConsumerCommitLogs(pTrans, pConsumerNew); TAOS_CHECK_GOTO(validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false), &lino, END);
if (code != 0) {
goto END; TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), &lino, END);
}
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
END: END:
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMqConsumerClearMsg *pClearMsg = pMsg->pCont; SMqConsumerClearMsg *pClearMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
@ -205,7 +200,7 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId); mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId);
return 0; TAOS_RETURN(code);
} }
mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId, mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId,
@ -213,21 +208,20 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL); pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL);
if (pConsumerNew == NULL){ if (pConsumerNew == NULL){
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm");
if (pTrans == NULL) { if (pTrans == NULL) {
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
// this is the drop action, not the update action // this is the drop action, not the update action
code = mndSetConsumerDropLogs(pTrans, pConsumerNew); TAOS_CHECK_GOTO(mndSetConsumerDropLogs(pTrans, pConsumerNew), &lino, END);
if (code != 0) {
goto END;
}
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
@ -235,7 +229,7 @@ END:
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) { static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) {
@ -310,10 +304,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
SMqHbRsp rsp = {0}; SMqHbRsp rsp = {0};
SMqConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) { TAOS_CHECK_GOTO(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req), NULL, end);
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
}
int64_t consumerId = req.consumerId; int64_t consumerId = req.consumerId;
pConsumer = mndAcquireConsumer(pMnode, consumerId); pConsumer = mndAcquireConsumer(pMnode, consumerId);
@ -322,10 +313,8 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST; code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto end; goto end;
} }
code = checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user);
if (code != 0) { TAOS_CHECK_GOTO(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user), NULL, end);
goto end;
}
atomic_store_32(&pConsumer->hbStatus, 0); atomic_store_32(&pConsumer->hbStatus, 0);
@ -343,7 +332,7 @@ end:
tDestroySMqHbRsp(&rsp); tDestroySMqHbRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDestroySMqHbReq(&req); tDestroySMqHbReq(&req);
return code; TAOS_RETURN(code);
} }
static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){
@ -431,6 +420,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
} }
static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoch, int64_t consumerId){ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoch, int64_t consumerId){
int32_t code = 0;
// encode rsp // encode rsp
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp); int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp);
void *buf = rpcMallocCont(tlen); void *buf = rpcMallocCont(tlen);
@ -452,7 +442,7 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc
// send rsp // send rsp
pMsg->info.rsp = buf; pMsg->info.rsp = buf;
pMsg->info.rspLen = tlen; pMsg->info.rspLen = tlen;
return 0; TAOS_RETURN(code);
} }
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
@ -461,9 +451,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
SMqAskEpRsp rsp = {0}; SMqAskEpRsp rsp = {0};
int32_t code = 0; int32_t code = 0;
if (tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req) < 0) { TAOS_CHECK_RETURN(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req));
return TSDB_CODE_OUT_OF_MEMORY;
}
int64_t consumerId = req.consumerId; int64_t consumerId = req.consumerId;
@ -517,19 +505,29 @@ END:
} }
int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static void freeItem(void *param) { static void freeItem(void *param) {
@ -585,6 +583,7 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb
} }
static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
int32_t code = 0;
taosArraySort(pTopicList, taosArrayCompareString); taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
@ -592,11 +591,10 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
for (int i = 0; i < newTopicNum; i++) { for (int i = 0; i < newTopicNum; i++) {
int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i)); int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i));
if (gNum >= MND_MAX_GROUP_PER_TOPIC) { if (gNum >= MND_MAX_GROUP_PER_TOPIC) {
terrno = TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE; return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
return -1;
} }
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){
@ -672,7 +670,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) { if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over; goto _over;
} }
@ -683,21 +682,21 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pConsumerNew = buildSubConsumer(pMnode, &subscribe); pConsumerNew = buildSubConsumer(pMnode, &subscribe);
if(pConsumerNew == NULL){ if(pConsumerNew == NULL){
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over; goto _over;
} }
code = mndSetConsumerCommitLogs(pTrans, pConsumerNew);
if (code != 0) goto _over;
code = mndTransPrepare(pMnode, pTrans); TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), NULL, _over);
if (code != 0) goto _over;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _over);
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
_over: _over:
mndTransDrop(pTrans); mndTransDrop(pTrans);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
return code; TAOS_RETURN(code);
} }
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {

685
source/dnode/mnode/impl/src/mndDb.c
View File

File diff suppressed because it is too large Load Diff

17
source/dnode/mnode/impl/src/mndMain.c
View File

@ -464,8 +464,7 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
static int32_t mndInitWal(SMnode *pMnode) { static int32_t mndInitWal(SMnode *pMnode) {
char path[PATH_MAX + 20] = {0}; char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = { SWalCfg cfg = {.vgId = 1,
.vgId = 1,
.fsyncPeriod = 0, .fsyncPeriod = 0,
.rollPeriod = -1, .rollPeriod = -1,
.segSize = -1, .segSize = -1,
@ -473,17 +472,15 @@ static int32_t mndInitWal(SMnode *pMnode) {
.retentionSize = 0, .retentionSize = 0,
.level = TAOS_WAL_FSYNC, .level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0, .encryptAlgorithm = 0,
.encryptKey = {0} .encryptKey = {0}};
};
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL) {
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0; cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL) ? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){ if (tsEncryptKey[0] == '\0') {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1; return -1;
} } else {
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
} }
} }
@ -856,7 +853,7 @@ _OVER:
return -1; return -1;
} }
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) { int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;

256
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -69,6 +69,7 @@ int32_t mndInitMnode(SMnode *pMnode) {
void mndCleanupMnode(SMnode *pMnode) {} void mndCleanupMnode(SMnode *pMnode) {}
SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) { SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
terrno = 0;
SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId); SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
@ -82,13 +83,18 @@ void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
} }
static int32_t mndCreateDefaultMnode(SMnode *pMnode) { static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
int32_t code = 0;
SMnodeObj mnodeObj = {0}; SMnodeObj mnodeObj = {0};
mnodeObj.id = 1; mnodeObj.id = 1;
mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.createdTime = taosGetTimestampMs();
mnodeObj.updateTime = mnodeObj.createdTime; mnodeObj.updateTime = mnodeObj.createdTime;
SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw); mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw);
@ -97,25 +103,27 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1; return -1;
} }
mInfo("trans:%d, used to create mnode:%d", pTrans->id, mnodeObj.id); mInfo("trans:%d, used to create mnode:%d", pTrans->id, mnodeObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; TAOS_RETURN(code);
} }
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) {
@ -188,16 +196,19 @@ _OVER:
} }
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
int32_t code = 0;
mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj); mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id); pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) { if (pObj->pDnode == NULL) {
mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
int32_t code = 0;
} }
pObj->syncState = TAOS_SYNC_STATE_OFFLINE; pObj->syncState = TAOS_SYNC_STATE_OFFLINE;
mndReloadSyncConfig(pSdb->pMnode); mndReloadSyncConfig(pSdb->pMnode);
return 0; TAOS_RETURN(code);
} }
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
@ -271,38 +282,59 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
} }
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
int32_t mndSetRestoreCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { int32_t mndSetRestoreCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) { static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
@ -315,15 +347,16 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p
.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED, .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq, static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq,
SEpSet *pAlterMnodeTypeEpSet) { SEpSet *pAlterMnodeTypeEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq);
@ -337,14 +370,15 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeType
.acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER, .acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
@ -357,15 +391,16 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA
.acceptableCode = 0, .acceptableCode = 0,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq);
@ -378,11 +413,11 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop
.acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED, .acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -426,9 +461,9 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -474,9 +509,9 @@ int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -517,9 +552,9 @@ static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -565,19 +600,23 @@ int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
SMnodeObj mnodeObj = {0}; SMnodeObj mnodeObj = {0};
mnodeObj.id = pDnode->id; mnodeObj.id = pDnode->id;
@ -586,8 +625,8 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeObj.role = TAOS_SYNC_ROLE_LEARNER; mnodeObj.role = TAOS_SYNC_ROLE_LEARNER;
mnodeObj.lastIndex = pMnode->applied; mnodeObj.lastIndex = pMnode->applied;
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj), NULL, _OVER);
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj), NULL, _OVER);
SMnodeObj mnodeLeaderObj = {0}; SMnodeObj mnodeLeaderObj = {0};
mnodeLeaderObj.id = pDnode->id; mnodeLeaderObj.id = pDnode->id;
@ -596,15 +635,15 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER; mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER;
mnodeLeaderObj.lastIndex = pMnode->applied + 1; mnodeLeaderObj.lastIndex = pMnode->applied + 1;
if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj), NULL, _OVER);
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
@ -614,19 +653,14 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateMnodeReq createReq = {0}; SMCreateMnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("mnode:%d, start to create", createReq.dnodeId); mInfo("mnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE), NULL, _OVER);
goto _OVER;
}
pObj = mndAcquireMnode(pMnode, createReq.dnodeId); pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
if (pObj != NULL) { if (pObj != NULL) {
terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; code = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) {
goto _OVER; goto _OVER;
@ -634,17 +668,17 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
terrno = TSDB_CODE_MND_TOO_MANY_MNODES; code = TSDB_CODE_MND_TOO_MANY_MNODES;
goto _OVER; goto _OVER;
} }
if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE; code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER; goto _OVER;
} }
@ -665,27 +699,38 @@ _OVER:
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj, static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj,
bool force) { bool force) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SDDropMnodeReq dropReq = {0}; SDDropMnodeReq dropReq = {0};
@ -700,32 +745,32 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
if (totalMnodes == 2) { if (totalMnodes == 2) {
if (force) { if (force) {
mError("cant't force drop dnode, since a mnode on it and replica is 2"); mError("cant't force drop dnode, since a mnode on it and replica is 2");
terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE; code = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
return -1; TAOS_RETURN(code);
} }
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
if (!force) { if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
} }
} else if (totalMnodes == 3) { } else if (totalMnodes == 3) {
mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes); mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
if (!force) { if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
} }
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
} else { } else {
return -1; TAOS_RETURN(-1);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) { int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
pObj->lastIndex = pMnode->applied; pObj->lastIndex = pMnode->applied;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force));
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj));
return 0; return 0;
} }
@ -734,19 +779,23 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) {
STrans *pTrans = NULL; STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
@ -755,38 +804,35 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
SMnodeObj *pObj = NULL; SMnodeObj *pObj = NULL;
SMDropMnodeReq dropReq = {0}; SMDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("mnode:%d, start to drop", dropReq.dnodeId); mInfo("mnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireMnode(pMnode, dropReq.dnodeId); pObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
if (pMnode->selfDnodeId == dropReq.dnodeId) { if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER; code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER; goto _OVER;
} }
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES; code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER; goto _OVER;
} }
if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) { if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE; code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER; goto _OVER;
} }
@ -805,7 +851,7 @@ _OVER:
mndReleaseMnode(pMnode, pObj); mndReleaseMnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
@ -892,13 +938,11 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
#if 1 #if 1
return 0; return 0;
#else #else
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0}; SDAlterMnodeReq alterReq = {0};
if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1}; SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas)); memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
@ -913,9 +957,9 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return 0; return 0;
} }
if (mndWriteFile(pMnode->path, &option) != 0) { if ((code = mndWriteFile(pMnode->path, &option)) != 0) {
mError("failed to write mnode file since %s", terrstr()); mError("failed to write mnode file since %s", terrstr());
return -1; TAOS_RETURN(code);
} }
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1}; SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
@ -939,14 +983,14 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
} }
} }
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) { if (code != 0) {
mError("failed to sync reconfig since %s", terrstr()); mError("failed to sync reconfig since %s", terrstr());
} else { } else {
mInfo("alter mnode sync success"); mInfo("alter mnode sync success");
} }
return code; TAOS_RETURN(code);
#endif #endif
} }

53
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -19,10 +19,11 @@
// connection/application/ // connection/application/
int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) { if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
@ -34,10 +35,11 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S
} }
*pDst = schema; *pDst = schema;
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
int32_t mndPerfsInitMeta(SHashObj *hash) { int32_t mndPerfsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0}; STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName)); tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -53,56 +55,56 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName)); tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName));
meta.numOfColumns = pSysDbTableMeta[i].colNum; meta.numOfColumns = pSysDbTableMeta[i].colNum;
if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) { TAOS_CHECK_RETURN(mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas));
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) { if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) { if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
*pRsp = *meta; *pRsp = *meta;
pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema));
return 0; TAOS_RETURN(code);
} }
int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) { if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
strcpy(pRsp->tbName, pMeta->tbName); strcpy(pRsp->tbName, pMeta->tbName);
@ -114,20 +116,21 @@ int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *t
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0; TAOS_RETURN(code);
} }
int32_t mndInitPerfs(SMnode *pMnode) { int32_t mndInitPerfs(SMnode *pMnode) {
int32_t code = 0;
pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pMnode->perfsMeta == NULL) { if (pMnode->perfsMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
return mndPerfsInitMeta(pMnode->perfsMeta); return mndPerfsInitMeta(pMnode->perfsMeta);

77
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -85,22 +85,23 @@ static void mndCancelGetNextApp(SMnode *pMnode, void *pIter);
static int32_t mndProcessSvrVerReq(SRpcMsg *pReq); static int32_t mndProcessSvrVerReq(SRpcMsg *pReq);
int32_t mndInitProfile(SMnode *pMnode) { int32_t mndInitProfile(SMnode *pMnode) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
// in ms // in ms
int32_t checkTime = tsShellActivityTimer * 2 * 1000; int32_t checkTime = tsShellActivityTimer * 2 * 1000;
pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->connCache == NULL) { if (pMgmt->connCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr()); mError("failed to alloc profile cache since %s", terrstr());
return -1; TAOS_RETURN(code);
} }
pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app"); pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app");
if (pMgmt->appCache == NULL) { if (pMgmt->appCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr()); mError("failed to alloc profile cache since %s", terrstr());
return -1; TAOS_RETURN(code);
} }
mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq); mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq);
@ -116,7 +117,7 @@ int32_t mndInitProfile(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp);
return 0; TAOS_RETURN(code);
} }
void mndCleanupProfile(SMnode *pMnode) { void mndCleanupProfile(SMnode *pMnode) {
@ -384,6 +385,7 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq)
static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); } static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); }
static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) { static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) {
terrno = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId)); SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId));
@ -431,13 +433,16 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) {
} }
static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) { static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) {
int32_t code = 0;
SAppHbReq *pReq = &pHbReq->app; SAppHbReq *pReq = &pHbReq->app;
SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId); SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId);
if (pApp == NULL) { if (pApp == NULL) {
pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq); pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq);
if (pApp == NULL) { if (pApp == NULL) {
mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr()); mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else { } else {
mDebug("a new app %" PRIx64 " is created", pReq->appId); mDebug("a new app %" PRIx64 " is created", pReq->appId);
mndReleaseApp(pMnode, pApp); mndReleaseApp(pMnode, pApp);
@ -475,6 +480,7 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) {
static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq,
SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) { SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
SRpcConnInfo connInfo = pMsg->info.conn; SRpcConnInfo connInfo = pMsg->info.conn;
@ -492,7 +498,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
pHbReq->app.pid, pHbReq->app.name, 0); pHbReq->app.pid, pHbReq->app.name, 0);
if (pConn == NULL) { if (pConn == NULL) {
mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr()); mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else { } else {
mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id);
} }
@ -501,9 +509,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic));
if (rspBasic == NULL) { if (rspBasic == NULL) {
mndReleaseConn(pMnode, pConn, true); mndReleaseConn(pMnode, pConn, true);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr()); mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr());
return -1; TAOS_RETURN(code);
} }
mndSaveQueryList(pConn, pBasic); mndSaveQueryList(pConn, pBasic);
@ -539,9 +547,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
hbRsp.info = taosArrayInit(kvNum, sizeof(SKv)); hbRsp.info = taosArrayInit(kvNum, sizeof(SKv));
if (NULL == hbRsp.info) { if (NULL == hbRsp.info) {
mError("taosArrayInit %d rsp kv failed", kvNum); mError("taosArrayInit %d rsp kv failed", kvNum);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
tFreeClientHbRsp(&hbRsp); tFreeClientHbRsp(&hbRsp);
return -1; TAOS_RETURN(code);
} }
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
@ -554,8 +562,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer); mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer);
SDynViewVersion *pRspVer = NULL; SDynViewVersion *pRspVer = NULL;
if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) { if (0 != (code = mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer))) {
return -1; TAOS_RETURN(code);
} }
if (needCheck) { if (needCheck) {
@ -647,13 +655,14 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
} }
static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SClientHbBatchReq batchReq = {0}; SClientHbBatchReq batchReq = {0};
if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) { if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) {
taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; TAOS_RETURN(code);
} }
SConnPreparedObj obj = {0}; SConnPreparedObj obj = {0};
@ -699,31 +708,27 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
taosArrayDestroy(obj.pQnodeList); taosArrayDestroy(obj.pQnodeList);
return 0; TAOS_RETURN(code);
} }
static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillQueryReq killReq = {0}; SKillQueryReq killReq = {0};
if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); mInfo("kill query msg is received, queryId:%s", killReq.queryStrId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY));
return -1;
}
int32_t connId = 0; int32_t connId = 0;
uint64_t queryId = 0; uint64_t queryId = 0;
char *p = strchr(killReq.queryStrId, ':'); char *p = strchr(killReq.queryStrId, ':');
if (NULL == p) { if (NULL == p) {
mError("invalid query id %s", killReq.queryStrId); mError("invalid query id %s", killReq.queryStrId);
terrno = TSDB_CODE_MND_INVALID_QUERY_ID; code = TSDB_CODE_MND_INVALID_QUERY_ID;
return -1; TAOS_RETURN(code);
} }
*p = 0; *p = 0;
connId = taosStr2Int32(killReq.queryStrId, NULL, 16); connId = taosStr2Int32(killReq.queryStrId, NULL, 16);
@ -732,40 +737,36 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t)); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t));
if (pConn == NULL) { if (pConn == NULL) {
mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId); mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID; code = TSDB_CODE_MND_INVALID_CONN_ID;
return -1; TAOS_RETURN(code);
} else { } else {
mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user); mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user);
pConn->killId = queryId; pConn->killId = queryId;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return 0; TAOS_RETURN(code);
} }
} }
static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { static int32_t mndProcessKillConnReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillConnReq killReq = {0}; SKillConnReq killReq = {0};
if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN));
return -1;
}
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t));
if (pConn == NULL) { if (pConn == NULL) {
mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); mError("connId:%u, failed to kill connection, conn not exist", killReq.connId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID; code = TSDB_CODE_MND_INVALID_CONN_ID;
return -1; TAOS_RETURN(code);
} else { } else {
mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user); mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user);
pConn->killed = 1; pConn->killed = 1;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
} }

190
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -138,15 +138,16 @@ _OVER:
} }
static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) { static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) {
int32_t code = 0;
mTrace("qnode:%d, perform insert action, row:%p", pObj->id, pObj); mTrace("qnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) { if (pObj->pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) {
@ -166,27 +167,42 @@ static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew
} }
static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) { bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
@ -194,6 +210,7 @@ bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
} }
int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0}; SDCreateQnodeReq createReq = {0};
createReq.dnodeId = pDnode->id; createReq.dnodeId = pDnode->id;
@ -212,23 +229,24 @@ int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeOb
action.msgType = TDMT_DND_CREATE_QNODE; action.msgType = TDMT_DND_CREATE_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED; action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0}; SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +257,12 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_QNODE; action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) { if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) {
@ -256,22 +274,26 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
qnodeObj.updateTime = qnodeObj.createdTime; qnodeObj.updateTime = qnodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
@ -281,19 +303,14 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateQnodeReq createReq = {0}; SMCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("qnode:%d, start to create", createReq.dnodeId); mInfo("qnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE), NULL, _OVER);
goto _OVER;
}
pObj = mndAcquireQnode(pMnode, createReq.dnodeId); pObj = mndAcquireQnode(pMnode, createReq.dnodeId);
if (pObj != NULL) { if (pObj != NULL) {
terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST; code = TSDB_CODE_MND_QNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) {
goto _OVER; goto _OVER;
@ -301,7 +318,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
@ -320,34 +337,47 @@ _OVER:
mndReleaseQnode(pMnode, pObj); mndReleaseQnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
if (mndTransAppendCommitlog(pTrans, pCommitRaw))
;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED))
;
TAOS_RETURN(code);
} }
static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0}; SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -358,20 +388,20 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
action.msgType = TDMT_DND_DROP_QNODE; action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) { int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropQnodeRedoLogs(pTrans, pObj));
if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropQnodeCommitLogs(pTrans, pObj));
if (!force) { if (!force) {
if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj));
} }
return 0; return 0;
} }
@ -380,18 +410,22 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
@ -400,23 +434,20 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
SQnodeObj *pObj = NULL; SQnodeObj *pObj = NULL;
SMDropQnodeReq dropReq = {0}; SMDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("qnode:%d, start to drop", dropReq.dnodeId); mInfo("qnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireQnode(pMnode, dropReq.dnodeId); pObj = mndAcquireQnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -435,10 +466,11 @@ _OVER:
mndReleaseQnode(pMnode, pObj); mndReleaseQnode(pMnode, pObj);
tFreeSDDropQnodeReq(&dropReq); tFreeSDDropQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SQnodeObj *pObj = NULL; SQnodeObj *pObj = NULL;
@ -447,8 +479,8 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
if (NULL == qnodeList) { if (NULL == qnodeList) {
mError("failed to alloc epSet while process qnode list req"); mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return terrno; TAOS_RETURN(code);
} }
while (1) { while (1) {
@ -484,20 +516,14 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
SQnodeListReq qlistReq = {0}; SQnodeListReq qlistReq = {0};
SQnodeListRsp qlistRsp = {0}; SQnodeListRsp qlistRsp = {0};
if (tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq), NULL, _OVER);
mError("failed to parse qnode list req");
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
if (mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum) != 0) { TAOS_CHECK_GOTO(mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum), NULL, _OVER);
goto _OVER;
}
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -509,7 +535,7 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
_OVER: _OVER:
tFreeSQnodeListRsp(&qlistRsp); tFreeSQnodeListRsp(&qlistRsp);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

6
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -88,7 +88,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
void *pRsp = NULL; void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("tDeserializeSBatchReq failed"); mError("tDeserializeSBatchReq failed");
goto _exit; goto _exit;
@ -119,7 +119,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req->msgType)]; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req->msgType)];
if (fp == NULL) { if (fp == NULL) {
mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
taosArrayDestroy(batchRsp.pRsps); taosArrayDestroy(batchRsp.pRsps);
return -1; return -1;
} }
@ -164,7 +164,7 @@ _exit:
taosArrayDestroyEx(batchReq.pMsgs, tFreeSBatchReqMsg); taosArrayDestroyEx(batchReq.pMsgs, tFreeSBatchReqMsg);
taosArrayDestroyEx(batchRsp.pRsps, mnodeFreeSBatchRspMsg); taosArrayDestroyEx(batchRsp.pRsps, mnodeFreeSBatchRspMsg);
return code; TAOS_RETURN(code);
} }
int32_t mndInitQuery(SMnode *pMnode) { int32_t mndInitQuery(SMnode *pMnode) {

145
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -50,17 +50,17 @@ static bool isCountWindowStreamTask(SSubplan* pPlan) {
int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
int64_t watermark, int64_t deleteMark) { int64_t watermark, int64_t deleteMark) {
int32_t code = 0;
SNode* pAst = NULL; SNode* pAst = NULL;
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
terrno = TSDB_CODE_SUCCESS;
if (nodesStringToNode(ast, &pAst) < 0) { if (nodesStringToNode(ast, &pAst) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
if (qSetSTableIdForRsma(pAst, uid) < 0) { if (qSetSTableIdForRsma(pAst, uid) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
@ -75,33 +75,33 @@ int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64
}; };
if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) { if (levelNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
int32_t opNum = LIST_LENGTH(inner->pNodeList); int32_t opNum = LIST_LENGTH(inner->pNodeList);
if (opNum != 1) { if (opNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
if (qSubPlanToString(plan, pDst, pDstLen) < 0) { if (qSubPlanToString(plan, pDst, pDstLen) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
END: END:
if (pAst) nodesDestroyNode(pAst); if (pAst) nodesDestroyNode(pAst);
if (pPlan) nodesDestroyNode((SNode*)pPlan); if (pPlan) nodesDestroyNode((SNode*)pPlan);
return terrno; TAOS_RETURN(code);
} }
int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
@ -127,6 +127,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList, int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList,
SStreamTask* pTask) { SStreamTask* pTask) {
int32_t code = 0;
bool isShuffle = false; bool isShuffle = false;
if (pStream->fixedSinkVgId == 0) { if (pStream->fixedSinkVgId == 0) {
@ -135,9 +136,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
isShuffle = true; isShuffle = true;
pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH;
pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH;
if (mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL) < 0) { TAOS_CHECK_RETURN(mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL));
return -1;
}
} }
sdbRelease(pMnode->pSdb, pDb); sdbRelease(pMnode->pSdb, pDb);
@ -166,7 +165,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
streamTaskSetFixedDownstreamInfo(pTask, pOneSinkTask); streamTaskSetFixedDownstreamInfo(pTask, pOneSinkTask);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) {
@ -242,11 +241,11 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, SStreamTask* pTask = NULL;
pStream->conf.fillHistory, pStream->subTableWithoutMd5); int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory,
if (pTask == NULL) { pStream->subTableWithoutMd5, &pTask);
terrno = TSDB_CODE_OUT_OF_MEMORY; if (code != 0) {
return terrno; return code;
} }
mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
@ -353,18 +352,13 @@ static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFi
} }
} }
static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) { static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, SStreamTask** pTask) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = int32_t code = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pTask);
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5); return code;
if (pTask == NULL) {
return NULL;
}
return pTask;
} }
static void addNewTaskList(SStreamObj* pStream) { static void addNewTaskList(SStreamObj* pStream) {
@ -398,10 +392,10 @@ static void setHTasksId(SStreamObj* pStream) {
static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey, static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) { SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) {
SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam); SStreamTask* pTask = NULL;
if (pTask == NULL) { int32_t code = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
terrno = TSDB_CODE_OUT_OF_MEMORY; if (code != TSDB_CODE_SUCCESS) {
return terrno; return code;
} }
mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory); mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
@ -412,10 +406,9 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId); streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup); code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; return code;
return terrno;
} }
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
@ -498,28 +491,29 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) { static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam,
SStreamTask** pAggTask) {
*pAggTask = NULL;
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid; uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks); SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pAggTask = int32_t code =
tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0, tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5); *pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pAggTask);
if (pAggTask == NULL) { return code;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
return pAggTask;
} }
static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup, static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup,
SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) { SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
int32_t code = 0; int32_t code = 0;
SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam); SStreamTask* pTask = NULL;
if (pTask == NULL) {
return terrno; code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
} }
if (pSnode != NULL) { if (pSnode != NULL) {
code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode); code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory); mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory);
@ -644,14 +638,15 @@ static void bindTwoLevel(SArray* tasks, int32_t begin, int32_t end) {
static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey, static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey,
SArray* pVerList) { SArray* pVerList) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb; SSdb* pSdb = pMnode->pSdb;
int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans); int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
bool hasExtraSink = false; bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
if (pDbObj == NULL) { if (pDbObj == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); bool multiTarget = (pDbObj->cfg.numOfVgroups > 1);
@ -675,9 +670,11 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
SSubplan* plan = getScanSubPlan(pPlan); // source plan SSubplan* plan = getScanSubPlan(pPlan); // source plan
if (plan == NULL) { if (plan == NULL) {
return terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int32_t code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1); code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -693,7 +690,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (numOfPlanLevel == 3) { if (numOfPlanLevel == 3) {
plan = getAggSubPlan(pPlan, 1); // middle agg plan plan = getAggSubPlan(pPlan, 1); // middle agg plan
if (plan == NULL) { if (plan == NULL) {
return terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
do { do {
SArray** list = taosArrayGetLast(pStream->tasks); SArray** list = taosArrayGetLast(pStream->tasks);
@ -720,7 +719,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
plan = getAggSubPlan(pPlan, 0); plan = getAggSubPlan(pPlan, 0);
if (plan == NULL) { if (plan == NULL) {
return terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
mDebug("doScheduleStream add final agg"); mDebug("doScheduleStream add final agg");
@ -729,7 +730,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
addNewTaskList(pStream); addNewTaskList(pStream);
code = addAggTask(pStream, pMnode, plan, pEpset, true); code = addAggTask(pStream, pMnode, plan, pEpset, true);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; TAOS_RETURN(code);
} }
bindTwoLevel(pStream->tasks, 0, size); bindTwoLevel(pStream->tasks, 0, size);
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
@ -740,26 +741,28 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
bindAggSink(pStream, pMnode, pStream->pHTasksList); bindAggSink(pStream, pMnode, pStream->pHTasksList);
} }
return TDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) { int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) {
int32_t code = 0;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
SEpSet mnodeEpset = {0}; SEpSet mnodeEpset = {0};
mndGetMnodeEpSet(pMnode, &mnodeEpset); mndGetMnodeEpSet(pMnode, &mnodeEpset);
int32_t code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList); code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
return code; TAOS_RETURN(code);
} }
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb; SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
@ -768,21 +771,21 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan); pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) { } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
SNode* pAst = NULL; SNode* pAst = NULL;
if (nodesStringToNode(pTopic->ast, &pAst) != 0) { if ((code = nodesStringToNode(pTopic->ast, &pAst)) != 0) {
mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
return -1; TAOS_RETURN(code);
} }
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) != 0) {
mError("failed to create topic:%s since %s", pTopic->name, terrstr()); mError("failed to create topic:%s since %s", pTopic->name, terrstr());
nodesDestroyNode(pAst); nodesDestroyNode(pAst);
return -1; TAOS_RETURN(code);
} }
nodesDestroyNode(pAst); nodesDestroyNode(pAst);
} }
@ -791,8 +794,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) { if (levelNum != 1) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1; TAOS_RETURN(code);
} }
SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
@ -800,8 +803,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList); int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) { if (opNum != 1) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1; TAOS_RETURN(code);
} }
pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0); pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
@ -836,13 +839,13 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) { if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
} else { } else {
pSub->qmsg = taosStrdup(""); pSub->qmsg = taosStrdup("");
} }
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
return 0; TAOS_RETURN(code);
} }

52
source/dnode/mnode/impl/test/sdb/sdbTest.cpp
View File

@ -387,7 +387,7 @@ TEST_F(MndTestSdb, 00_API) {
void *pRow2 = sdbGetRowObj(NULL); void *pRow2 = sdbGetRowObj(NULL);
ASSERT_EQ(pRow2 == NULL, 1); ASSERT_EQ(pRow2 == NULL, 1);
//sdbRaw.c // sdbRaw.c
SStrObj strObj; SStrObj strObj;
SSdbRaw *pRaw1 = NULL; SSdbRaw *pRaw1 = NULL;
strSetDefault(&strObj, 1); strSetDefault(&strObj, 1);
@ -399,32 +399,32 @@ TEST_F(MndTestSdb, 00_API) {
SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128); SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128);
ASSERT_EQ(pRaw2 == NULL, 1); ASSERT_EQ(pRaw2 == NULL, 1);
ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), -1); ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), -1); ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawDataLen(NULL, 0), -1); ASSERT_EQ(sdbSetRawDataLen(NULL, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), -1); ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), -1); ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), -1); ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), TSDB_CODE_INVALID_PARA);
ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), -1); ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), -1); ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), -1); ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawTotalSize(NULL), -1); ASSERT_EQ(sdbGetRawTotalSize(NULL), -1);
// sdbHash.c // sdbHash.c

15
source/dnode/mnode/sdb/src/sdb.c
View File

@ -112,6 +112,8 @@ void sdbCleanup(SSdb *pSdb) {
} }
int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
int32_t code = 0;
ESdbType sdbType = table.sdbType; ESdbType sdbType = table.sdbType;
EKeyType keyType = table.keyType; EKeyType keyType = table.keyType;
pSdb->keyTypes[sdbType] = table.keyType; pSdb->keyTypes[sdbType] = table.keyType;
@ -145,16 +147,17 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
} }
static int32_t sdbCreateDir(SSdb *pSdb) { static int32_t sdbCreateDir(SSdb *pSdb) {
int32_t code = 0;
if (taosMulMkDir(pSdb->currDir) != 0) { if (taosMulMkDir(pSdb->currDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, terrstr()); mError("failed to create dir:%s since %s", pSdb->currDir, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
if (taosMkDir(pSdb->tmpDir) != 0) { if (taosMkDir(pSdb->tmpDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->tmpDir, terrstr()); mError("failed to create dir:%s since %s", pSdb->tmpDir, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
return 0; return 0;

220
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -14,18 +14,19 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "crypt.h"
#include "sdb.h" #include "sdb.h"
#include "sync.h" #include "sync.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "wal.h"
#include "tglobal.h" #include "tglobal.h"
#include "crypt.h" #include "wal.h"
#define SDB_TABLE_SIZE 24 #define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512 #define SDB_RESERVE_SIZE 512
#define SDB_FILE_VER 1 #define SDB_FILE_VER 1
static int32_t sdbDeployData(SSdb *pSdb) { static int32_t sdbDeployData(SSdb *pSdb) {
int32_t code = 0;
mInfo("start to deploy sdb"); mInfo("start to deploy sdb");
for (int32_t i = SDB_MAX - 1; i >= 0; --i) { for (int32_t i = SDB_MAX - 1; i >= 0; --i) {
@ -33,8 +34,9 @@ static int32_t sdbDeployData(SSdb *pSdb) {
if (fp == NULL) continue; if (fp == NULL) continue;
mInfo("start to deploy sdb:%s", sdbTableName(i)); mInfo("start to deploy sdb:%s", sdbTableName(i));
if ((*fp)(pSdb->pMnode) != 0) { code = (*fp)(pSdb->pMnode);
mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr()); if (code != 0) {
mError("failed to deploy sdb:%s since %s", sdbTableName(i), tstrerror(code));
return -1; return -1;
} }
} }
@ -80,61 +82,62 @@ static void sdbResetData(SSdb *pSdb) {
} }
static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t code = 0;
int64_t sver = 0; int64_t sver = 0;
int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t)); int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
if (sver != SDB_FILE_VER) { if (sver != SDB_FILE_VER) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
ret = taosReadFile(pFile, &pSdb->applyIndex, sizeof(int64_t)); ret = taosReadFile(pFile, &pSdb->applyIndex, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
ret = taosReadFile(pFile, &pSdb->applyTerm, sizeof(int64_t)); ret = taosReadFile(pFile, &pSdb->applyTerm, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
ret = taosReadFile(pFile, &pSdb->applyConfig, sizeof(int64_t)); ret = taosReadFile(pFile, &pSdb->applyConfig, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = 0; int64_t maxId = 0;
ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
if (i < SDB_MAX) { if (i < SDB_MAX) {
pSdb->maxId[i] = maxId; pSdb->maxId[i] = maxId;
@ -145,12 +148,12 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int64_t ver = 0; int64_t ver = 0;
ret = taosReadFile(pFile, &ver, sizeof(int64_t)); ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
if (i < SDB_MAX) { if (i < SDB_MAX) {
pSdb->tableVer[i] = ver; pSdb->tableVer[i] = ver;
@ -160,37 +163,38 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
char reserve[SDB_RESERVE_SIZE] = {0}; char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(pFile, reserve, sizeof(reserve)); ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(reserve)) { if (ret != sizeof(reserve)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
return 0; return 0;
} }
static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t code = 0;
int64_t sver = SDB_FILE_VER; int64_t sver = SDB_FILE_VER;
if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (taosWriteFile(pFile, &pSdb->applyTerm, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->applyTerm, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (taosWriteFile(pFile, &pSdb->applyConfig, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->applyConfig, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
@ -199,8 +203,8 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
maxId = pSdb->maxId[i]; maxId = pSdb->maxId[i];
} }
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
} }
@ -210,15 +214,15 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
ver = pSdb->tableVer[i]; ver = pSdb->tableVer[i];
} }
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
} }
char reserve[SDB_RESERVE_SIZE] = {0}; char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
return 0; return 0;
@ -237,21 +241,22 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100); SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100);
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since %s", terrstr()); mError("failed read sdb file since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
taosMemoryFree(pRaw); taosMemoryFree(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mInfo("read sdb file:%s finished since %s", file, terrstr()); mInfo("read sdb file:%s finished since %s", file, tstrerror(code));
return 0; return 0;
} }
if (sdbReadFileHead(pSdb, pFile) != 0) { code = sdbReadFileHead(pSdb, pFile);
mError("failed to read sdb file:%s head since %s", file, terrstr()); if (code != 0) {
mError("failed to read sdb file:%s head since %s", file, tstrerror(code));
taosMemoryFree(pRaw); taosMemoryFree(pRaw);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; return -1;
@ -278,14 +283,14 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
} }
readLen = pRaw->dataLen + sizeof(int32_t); readLen = pRaw->dataLen + sizeof(int32_t);
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t); readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t);
} }
if (readLen >= bufLen) { if (readLen >= bufLen) {
bufLen = pRaw->dataLen * 2; bufLen = pRaw->dataLen * 2;
SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100);
if (pNewRaw == NULL) { if (pNewRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen); mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen);
goto _OVER; goto _OVER;
} }
@ -308,10 +313,14 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
goto _OVER; goto _OVER;
} }
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
int32_t count = 0; int32_t count = 0;
char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen)); char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen));
if (plantContent == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
SCryptOpts opts; SCryptOpts opts;
opts.len = ENCRYPTED_LEN(pRaw->dataLen); opts.len = ENCRYPTED_LEN(pRaw->dataLen);
@ -322,7 +331,7 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
count = CBC_Decrypt(&opts); count = CBC_Decrypt(&opts);
//mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__); // mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
memcpy(pRaw->pData, plantContent, pRaw->dataLen); memcpy(pRaw->pData, plantContent, pRaw->dataLen);
taosMemoryFree(plantContent); taosMemoryFree(plantContent);
@ -355,8 +364,7 @@ _OVER:
taosCloseFile(&pFile); taosCloseFile(&pFile);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
terrno = code; TAOS_RETURN(code);
return code;
} }
int32_t sdbReadFile(SSdb *pSdb) { int32_t sdbReadFile(SSdb *pSdb) {
@ -365,7 +373,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
sdbResetData(pSdb); sdbResetData(pSdb);
int32_t code = sdbReadFileImp(pSdb); int32_t code = sdbReadFileImp(pSdb);
if (code != 0) { if (code != 0) {
mError("failed to read sdb file since %s", terrstr()); mError("failed to read sdb file since %s", tstrerror(code));
sdbResetData(pSdb); sdbResetData(pSdb);
} }
@ -388,13 +396,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s for write since %s", tmpfile, terrstr()); mError("failed to open sdb file:%s for write since %s", tmpfile, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
if (sdbWriteFileHead(pSdb, pFile) != 0) { code = sdbWriteFileHead(pSdb, pFile);
mError("failed to write sdb file:%s head since %s", tmpfile, terrstr()); if (code != 0) {
mError("failed to write sdb file:%s head since %s", tmpfile, tstrerror(code));
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; return -1;
} }
@ -436,8 +445,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
} }
int32_t newDataLen = pRaw->dataLen; int32_t newDataLen = pRaw->dataLen;
char* newData = pRaw->pData; char *newData = pRaw->pData;
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
newDataLen = ENCRYPTED_LEN(pRaw->dataLen); newDataLen = ENCRYPTED_LEN(pRaw->dataLen);
newData = taosMemoryMalloc(newDataLen); newData = taosMemoryMalloc(newDataLen);
if (newData == NULL) { if (newData == NULL) {
@ -456,7 +465,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
int32_t count = CBC_Encrypt(&opts); int32_t count = CBC_Encrypt(&opts);
//mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s", // mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
// newDataLen, pRaw->dataLen, __FUNCTION__); // newDataLen, pRaw->dataLen, __FUNCTION__);
} }
@ -467,7 +476,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
break; break;
} }
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
taosMemoryFree(newData); taosMemoryFree(newData);
} }
@ -549,19 +558,22 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
} }
} }
if (code != 0) { if (code != 0) {
mError("failed to write sdb file since %s", terrstr()); mError("failed to write sdb file since %s", tstrerror(code));
} }
taosThreadMutexUnlock(&pSdb->filelock); taosThreadMutexUnlock(&pSdb->filelock);
return code; return code;
} }
int32_t sdbDeploy(SSdb *pSdb) { int32_t sdbDeploy(SSdb *pSdb) {
if (sdbDeployData(pSdb) != 0) { int32_t code = 0;
return -1; code = sdbDeployData(pSdb);
if (code != 0) {
TAOS_RETURN(code);
} }
if (sdbWriteFile(pSdb, 0) != 0) { code = sdbWriteFile(pSdb, 0);
return -1; if (code != 0) {
TAOS_RETURN(code);
} }
return 0; return 0;
@ -605,6 +617,7 @@ static void sdbCloseIter(SSdbIter *pIter) {
} }
int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config) { int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config) {
int32_t code = 0;
SSdbIter *pIter = sdbCreateIter(pSdb); SSdbIter *pIter = sdbCreateIter(pSdb);
if (pIter == NULL) return -1; if (pIter == NULL) return -1;
@ -617,19 +630,19 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter
int64_t commitConfig = pSdb->commitConfig; int64_t commitConfig = pSdb->commitConfig;
if (taosCopyFile(datafile, pIter->name) < 0) { if (taosCopyFile(datafile, pIter->name) < 0) {
taosThreadMutexUnlock(&pSdb->filelock); taosThreadMutexUnlock(&pSdb->filelock);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, terrstr()); mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, tstrerror(code));
sdbCloseIter(pIter); sdbCloseIter(pIter);
return -1; TAOS_RETURN(code);
} }
taosThreadMutexUnlock(&pSdb->filelock); taosThreadMutexUnlock(&pSdb->filelock);
pIter->file = taosOpenFile(pIter->name, TD_FILE_READ); pIter->file = taosOpenFile(pIter->name, TD_FILE_READ);
if (pIter->file == NULL) { if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s since %s", pIter->name, terrstr()); mError("failed to open sdb file:%s since %s", pIter->name, tstrerror(code));
sdbCloseIter(pIter); sdbCloseIter(pIter);
return -1; TAOS_RETURN(code);
} }
*ppIter = pIter; *ppIter = pIter;
@ -645,21 +658,22 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter
void sdbStopRead(SSdb *pSdb, SSdbIter *pIter) { sdbCloseIter(pIter); } void sdbStopRead(SSdb *pSdb, SSdbIter *pIter) { sdbCloseIter(pIter); }
int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) { int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
int32_t code = 0;
int32_t maxlen = 4096; int32_t maxlen = 4096;
void *pBuf = taosMemoryCalloc(1, maxlen); void *pBuf = taosMemoryCalloc(1, maxlen);
if (pBuf == NULL) { if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen); int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen);
if (readlen < 0 || readlen > maxlen) { if (readlen < 0 || readlen > maxlen) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, terrstr(), pIter->total); mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, tstrerror(code), pIter->total);
*ppBuf = NULL; *ppBuf = NULL;
*len = 0; *len = 0;
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
return -1; TAOS_RETURN(code);
} else if (readlen == 0) { } else if (readlen == 0) {
mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total); mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total);
*ppBuf = NULL; *ppBuf = NULL;
@ -676,15 +690,19 @@ int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
} }
int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
int32_t code = 0;
SSdbIter *pIter = sdbCreateIter(pSdb); SSdbIter *pIter = sdbCreateIter(pSdb);
if (pIter == NULL) return -1; if (pIter == NULL) {
code = terrno;
TAOS_RETURN(code);
}
pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pIter->file == NULL) { if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open %s since %s", pIter->name, terrstr()); mError("failed to open %s since %s", pIter->name, tstrerror(code));
sdbCloseIter(pIter); sdbCloseIter(pIter);
return -1; TAOS_RETURN(code);
} }
*ppIter = pIter; *ppIter = pIter;
@ -702,8 +720,8 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
} }
if (taosFsyncFile(pIter->file) != 0) { if (taosFsyncFile(pIter->file) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, terrstr()); mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -713,13 +731,14 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
char datafile[PATH_MAX] = {0}; char datafile[PATH_MAX] = {0};
snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
if (taosRenameFile(pIter->name, datafile) != 0) { if (taosRenameFile(pIter->name, datafile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, terrstr()); mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, tstrerror(code));
goto _OVER; goto _OVER;
} }
if (sdbReadFile(pSdb) != 0) { code = sdbReadFile(pSdb);
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr()); if (code != 0) {
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -742,11 +761,12 @@ _OVER:
} }
int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) { int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) {
int32_t code = 0;
int32_t writelen = taosWriteFile(pIter->file, pBuf, len); int32_t writelen = taosWriteFile(pIter->file, pBuf, len);
if (writelen != len) { if (writelen != len) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write len:%d since %s, total:%" PRId64, len, terrstr(), pIter->total); mError("failed to write len:%d since %s, total:%" PRId64, len, tstrerror(code), pIter->total);
return -1; TAOS_RETURN(code);
} }
pIter->total += writelen; pIter->total += writelen;

131
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -53,14 +53,15 @@ void sdbFreeRaw(SSdbRaw *pRaw) {
} }
int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int8_t) > pRaw->dataLen) { if (dataPos + sizeof(int8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int8_t *)(pRaw->pData + dataPos) = val; *(int8_t *)(pRaw->pData + dataPos) = val;
@ -68,14 +69,15 @@ int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
} }
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) { int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) { if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(uint8_t *)(pRaw->pData + dataPos) = val; *(uint8_t *)(pRaw->pData + dataPos) = val;
@ -83,14 +85,15 @@ int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
} }
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) { int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int32_t) > pRaw->dataLen) { if (dataPos + sizeof(int32_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int32_t *)(pRaw->pData + dataPos) = val; *(int32_t *)(pRaw->pData + dataPos) = val;
@ -98,14 +101,15 @@ int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
} }
int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) { int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int16_t) > pRaw->dataLen) { if (dataPos + sizeof(int16_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int16_t *)(pRaw->pData + dataPos) = val; *(int16_t *)(pRaw->pData + dataPos) = val;
@ -113,14 +117,15 @@ int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) {
} }
int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) { int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int64_t) > pRaw->dataLen) { if (dataPos + sizeof(int64_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int64_t *)(pRaw->pData + dataPos) = val; *(int64_t *)(pRaw->pData + dataPos) = val;
@ -128,14 +133,15 @@ int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) {
} }
int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) { int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + valLen > pRaw->dataLen) { if (dataPos + valLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
if (pVal != NULL) { if (pVal != NULL) {
@ -145,14 +151,15 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
} }
int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) { int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataLen > pRaw->dataLen) { if (dataLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
pRaw->dataLen = dataLen; pRaw->dataLen = dataLen;
@ -160,14 +167,15 @@ int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) {
} }
int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) { int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (status == SDB_STATUS_INIT) { if (status == SDB_STATUS_INIT) {
terrno = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
return -1; TAOS_RETURN(code);
} }
pRaw->status = status; pRaw->status = status;
@ -175,14 +183,15 @@ int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) {
} }
int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) { int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int8_t) > pRaw->dataLen) { if (dataPos + sizeof(int8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int8_t *)(pRaw->pData + dataPos); *val = *(int8_t *)(pRaw->pData + dataPos);
@ -190,14 +199,15 @@ int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
} }
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) { int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) { if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(uint8_t *)(pRaw->pData + dataPos); *val = *(uint8_t *)(pRaw->pData + dataPos);
@ -205,14 +215,15 @@ int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
} }
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) { int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int32_t) > pRaw->dataLen) { if (dataPos + sizeof(int32_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int32_t *)(pRaw->pData + dataPos); *val = *(int32_t *)(pRaw->pData + dataPos);
@ -220,14 +231,15 @@ int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
} }
int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) { int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int16_t) > pRaw->dataLen) { if (dataPos + sizeof(int16_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int16_t *)(pRaw->pData + dataPos); *val = *(int16_t *)(pRaw->pData + dataPos);
@ -235,14 +247,15 @@ int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) {
} }
int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) { int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int64_t) > pRaw->dataLen) { if (dataPos + sizeof(int64_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int64_t *)(pRaw->pData + dataPos); *val = *(int64_t *)(pRaw->pData + dataPos);
@ -250,14 +263,15 @@ int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) {
} }
int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) { int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + valLen > pRaw->dataLen) { if (dataPos + valLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
if (pVal != NULL) { if (pVal != NULL) {
memcpy(pVal, pRaw->pData + dataPos, valLen); memcpy(pVal, pRaw->pData + dataPos, valLen);
@ -266,9 +280,10 @@ int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valL
} }
int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver) { int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
*sver = pRaw->sver; *sver = pRaw->sver;

16
source/dnode/snode/src/snode.c
View File

@ -41,7 +41,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
SCheckpointInfo *pChkInfo = &pTask->chkInfo; SCheckpointInfo *pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
char *p = streamTaskGetStatus(pTask)->name; char *p = streamTaskGetStatus(pTask).name;
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
@ -61,6 +61,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
} }
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
int32_t code = 0;
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) { if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -68,12 +69,17 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
} }
stopRsync(); stopRsync();
startRsync(); code = startRsync();
if (code != 0) {
terrno = code;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE,
if (pSnode->pMeta == NULL) { taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta);
terrno = TSDB_CODE_OUT_OF_MEMORY; if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto FAIL; goto FAIL;
} }

5
source/dnode/vnode/CMakeLists.txt
View File

@ -68,11 +68,8 @@ set(
"src/tq/tqOffset.c" "src/tq/tqOffset.c"
"src/tq/tqPush.c" "src/tq/tqPush.c"
"src/tq/tqSink.c" "src/tq/tqSink.c"
"src/tq/tqCommit.c"
"src/tq/tqStreamTask.c" "src/tq/tqStreamTask.c"
"src/tq/tqHandleSnapshot.c" "src/tq/tqSnapshot.c"
"src/tq/tqCheckInfoSnapshot.c"
"src/tq/tqOffsetSnapshot.c"
"src/tq/tqStreamStateSnap.c" "src/tq/tqStreamStateSnap.c"
"src/tq/tqStreamTaskSnap.c" "src/tq/tqStreamTaskSnap.c"

49
source/dnode/vnode/src/inc/tq.h
View File

@ -41,8 +41,6 @@ extern "C" {
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on // clang-format on
typedef struct STqOffsetStore STqOffsetStore;
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
// tqExec // tqExec
@ -101,10 +99,11 @@ struct STQ {
SHashObj* pPushMgr; // subKey -> STqHandle SHashObj* pPushMgr; // subKey -> STqHandle
SHashObj* pHandle; // subKey -> STqHandle SHashObj* pHandle; // subKey -> STqHandle
SHashObj* pCheckInfo; // topic -> SAlterCheckInfo SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
STqOffsetStore* pOffsetStore; SHashObj* pOffset; // subKey -> STqOffsetVal
TDB* pMetaDB; TDB* pMetaDB;
TTB* pExecStore; TTB* pExecStore;
TTB* pCheckStore; TTB* pCheckStore;
TTB* pOffsetStore;
SStreamMeta* pStreamMeta; SStreamMeta* pStreamMeta;
}; };
@ -128,29 +127,22 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
int32_t tqMetaOpen(STQ* pTq); int32_t tqMetaOpen(STQ* pTq);
int32_t tqMetaClose(STQ* pTq); int32_t tqMetaClose(STQ* pTq);
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen);
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen); int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen);
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle);
int32_t tqMetaRestoreCheckInfo(STQ* pTq); int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen);
int32_t tqMetaGetHandle(STQ* pTq, const char* key); int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen);
int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle);
void* tqMetaGetOffset(STQ* pTq, const char* subkey);
STqOffsetStore* tqOffsetOpen(STQ* pTq);
int32_t tqMetaTransform(STQ* pTq); int32_t tqMetaTransform(STQ* pTq);
void tqOffsetClose(STqOffsetStore*);
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset);
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetCommitFile(STqOffsetStore* pStore);
// tqSink // tqSink
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule); const char* pIdStr, bool newSubTableRule);
void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data);
// tqOffset // tqOffset
char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqBuildFName(char** data, const char* path, char* name);
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name);
// tq util // tq util
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
@ -166,6 +158,25 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule);
#define TQ_ERR_GO_TO_END(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
goto END; \
} \
} while (0)
#define TQ_ERR_RETURN(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
return code; \
} \
} while (0)
#define TQ_SUBSCRIBE_NAME "subscribe"
#define TQ_OFFSET_NAME "offset-ver0"
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -258,7 +258,6 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT; typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile); int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);

33
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -70,10 +70,6 @@ typedef struct STsdbSnapRAWReader STsdbSnapRAWReader;
typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter; typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter;
typedef struct STqSnapReader STqSnapReader; typedef struct STqSnapReader STqSnapReader;
typedef struct STqSnapWriter STqSnapWriter; typedef struct STqSnapWriter STqSnapWriter;
typedef struct STqOffsetReader STqOffsetReader;
typedef struct STqOffsetWriter STqOffsetWriter;
typedef struct STqCheckInfoReader STqCheckInfoReader;
typedef struct STqCheckInfoWriter STqCheckInfoWriter;
typedef struct SStreamTaskReader SStreamTaskReader; typedef struct SStreamTaskReader SStreamTaskReader;
typedef struct SStreamTaskWriter SStreamTaskWriter; typedef struct SStreamTaskWriter SStreamTaskWriter;
typedef struct SStreamStateReader SStreamStateReader; typedef struct SStreamStateReader SStreamStateReader;
@ -222,8 +218,8 @@ typedef struct SMetaInfo {
int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader); int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
// tsdb // tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force); int32_t tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force);
int tsdbClose(STsdb** pTsdb); int32_t tsdbClose(STsdb** pTsdb);
int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbBegin(STsdb* pTsdb);
// int32_t tsdbPrepareCommit(STsdb* pTsdb); // int32_t tsdbPrepareCommit(STsdb* pTsdb);
// int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
@ -266,7 +262,6 @@ int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver); int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq); int32_t tqScanWal(STQ* pTq);
int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId);
// tq-mq // tq-mq
@ -349,29 +344,17 @@ int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr);
int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter); int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter);
int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback); int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback);
// STqSnapshotReader == // STqSnapshotReader ==
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader);
int32_t tqSnapReaderClose(STqSnapReader** ppReader); int32_t tqSnapReaderClose(STqSnapReader** ppReader);
int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData); int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData);
// STqSnapshotWriter ====================================== // STqSnapshotWriter ======================================
int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter); int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter);
int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback); int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback);
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
// STqCheckInfoshotReader ==
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader); int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader); int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData);
// STqCheckInfoshotWriter ======================================
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter);
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback);
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData);
// STqOffsetReader ========================================
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader);
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader);
int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData);
// STqOffsetWriter ========================================
int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter);
int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback);
int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData);
// SStreamTaskWriter ====================================== // SStreamTaskWriter ======================================
int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader); int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader);

19
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -297,8 +297,17 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG); sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG);
pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta);
tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask);
pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); int32_t code = streamCreateStateMachine(pStreamTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = streamTaskCreateActiveChkptInfo(&pStreamTask->chkInfo.pActiveInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId); pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId);
if (!pStreamState) { if (!pStreamState) {
terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
@ -414,7 +423,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
goto _err; goto _err;
} }
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != 0) {
goto _err; goto _err;
} }
@ -540,12 +549,12 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid)
taosArrayDestroy(pUidArray); taosArrayDestroy(pUidArray);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
} else { } else {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }

121
source/dnode/vnode/src/tq/tq.c
View File

@ -76,9 +76,12 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
taosInitRWLatch(&pTq->lock); taosInitRWLatch(&pTq->lock);
pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK);
pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); pTq->pCheckInfo = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo);
pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pOffset, (FDelete)tDeleteSTqOffset);
int32_t code = tqInitialize(pTq); int32_t code = tqInitialize(pTq);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqClose(pTq); tqClose(pTq);
@ -90,26 +93,19 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
int32_t tqInitialize(STQ* pTq) { int32_t tqInitialize(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
pTq->pStreamMeta =
streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback); int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1,
if (pTq->pStreamMeta == NULL) { tqStartTaskCompleteCallback, &pTq->pStreamMeta);
return -1; if (code != TSDB_CODE_SUCCESS) {
return code;
} }
streamMetaLoadAllTasks(pTq->pStreamMeta); streamMetaLoadAllTasks(pTq->pStreamMeta);
if (tqMetaTransform(pTq) < 0) { if (tqMetaOpen(pTq) < 0) {
return -1; return -1;
} }
if (tqMetaRestoreCheckInfo(pTq) < 0) {
return -1;
}
pTq->pOffsetStore = tqOffsetOpen(pTq);
if (pTq->pOffsetStore == NULL) {
return -1;
}
return 0; return 0;
} }
@ -133,10 +129,10 @@ void tqClose(STQ* pTq) {
pIter = taosHashIterate(pTq->pPushMgr, pIter); pIter = taosHashIterate(pTq->pPushMgr, pIter);
} }
tqOffsetClose(pTq->pOffsetStore);
taosHashCleanup(pTq->pHandle); taosHashCleanup(pTq->pHandle);
taosHashCleanup(pTq->pPushMgr); taosHashCleanup(pTq->pPushMgr);
taosHashCleanup(pTq->pCheckInfo); taosHashCleanup(pTq->pCheckInfo);
taosHashCleanup(pTq->pOffset);
taosMemoryFree(pTq->path); taosMemoryFree(pTq->path);
tqMetaClose(pTq); tqMetaClose(pTq);
@ -221,7 +217,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
goto end; goto end;
} }
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, pOffset->subKey);
if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) { if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) {
tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64,
vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version);
@ -229,10 +225,14 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
} }
// save the new offset value // save the new offset value
code = tqOffsetWrite(pTq->pOffsetStore, pOffset); if (taosHashPut(pTq->pOffset, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset))){
if(code != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_INVALID_MSG; return -1;
goto end; }
if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), msg, msgLen - sizeof(vgOffset.consumerId)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
return 0; return 0;
@ -360,21 +360,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
while (1) { while (1) {
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
// 1. find handle // 1. find handle
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); code = tqMetaGetHandle(pTq, req.subKey, &pHandle);
if (pHandle == NULL) { if (code != TDB_CODE_SUCCESS) {
do {
if (tqMetaGetHandle(pTq, req.subKey) == 0) {
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle != NULL) {
break;
}
}
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
code = -1; return -1;
goto END;
} while (0);
} }
// 2. check rebalance status // 2. check rebalance status
@ -443,7 +434,7 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, vgOffset.offset.subKey); STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, vgOffset.offset.subKey);
if (pSavedOffset == NULL) { if (pSavedOffset == NULL) {
terrno = TSDB_CODE_TMQ_NO_COMMITTED; terrno = TSDB_CODE_TMQ_NO_COMMITTED;
return terrno; return terrno;
@ -523,7 +514,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
if (reqOffset.type == TMQ_OFFSET__LOG) { if (reqOffset.type == TMQ_OFFSET__LOG) {
dataRsp.common.rspOffset.version = reqOffset.version; dataRsp.common.rspOffset.version = reqOffset.version;
} else if (reqOffset.type < 0) { } else if (reqOffset.type < 0) {
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); STqOffset* pOffset = (STqOffset*)(STqOffset*)tqMetaGetOffset(pTq, req.subKey);
if (pOffset != NULL) { if (pOffset != NULL) {
if (pOffset->val.type != TMQ_OFFSET__LOG) { if (pOffset->val.type != TMQ_OFFSET__LOG) {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey); tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey);
@ -590,12 +581,14 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} }
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey); if (taosHashRemove(pTq->pOffset, pReq->subKey, strlen(pReq->subKey)) != 0) {
if (code != 0) { tqError("cannot process tq delete req %s, since no such offset in hash", pReq->subKey);
tqError("cannot process tq delete req %s, since no such offset in cache", pReq->subKey); }
if (tqMetaDeleteInfo(pTq, pTq->pOffsetStore, pReq->subKey, strlen(pReq->subKey)) != 0) {
tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey);
} }
if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { if (tqMetaDeleteInfo(pTq, pTq->pExecStore, pReq->subKey, strlen(pReq->subKey)) < 0) {
tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey); tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey);
} }
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
@ -605,18 +598,16 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
STqCheckInfo info = {0}; STqCheckInfo info = {0};
SDecoder decoder; if(tqMetaDecodeCheckInfo(&info, msg, msgLen) != 0){
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
if (tDecodeSTqCheckInfo(&decoder, &info) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
tDecoderClear(&decoder);
if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) { if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSTqCheckInfo(&info);
return -1; return -1;
} }
if (tqMetaSaveCheckInfo(pTq, info.topic, msg, msgLen) < 0) { if (tqMetaSaveInfo(pTq, pTq->pCheckStore, info.topic, strlen(info.topic), msg, msgLen) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -628,7 +619,8 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
if (tqMetaDeleteCheckInfo(pTq, msg) < 0) { if (tqMetaDeleteInfo(pTq, pTq->pCheckStore, msg, strlen(msg)) < 0) {
tqError("cannot process tq delete check info req %s, since no such check info", msg);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -652,19 +644,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId); req.oldConsumerId, req.newConsumerId);
STqHandle* pHandle = NULL;
while (1) {
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle) {
break;
}
taosRLockLatch(&pTq->lock); taosRLockLatch(&pTq->lock);
ret = tqMetaGetHandle(pTq, req.subKey); STqHandle* pHandle = NULL;
(void)tqMetaGetHandle(pTq, req.subKey, &pHandle); //ignore return code
taosRUnLockLatch(&pTq->lock); taosRUnLockLatch(&pTq->lock);
if (ret < 0) {
break;
}
}
if (pHandle == NULL) { if (pHandle == NULL) {
if (req.oldConsumerId != -1) { if (req.oldConsumerId != -1) {
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
@ -675,7 +658,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
goto end; goto end;
} }
STqHandle handle = {0}; STqHandle handle = {0};
ret = tqCreateHandle(pTq, &req, &handle); ret = tqMetaCreateHandle(pTq, &req, &handle);
if (ret < 0) { if (ret < 0) {
tqDestroyTqHandle(&handle); tqDestroyTqHandle(&handle);
goto end; goto end;
@ -767,7 +750,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
SCheckpointInfo* pChkInfo = &pTask->chkInfo; SCheckpointInfo* pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask); tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name; char* p = streamTaskGetStatus(pTask).name;
const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
@ -856,7 +839,8 @@ int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) {
SStreamMeta* pMeta = pStreamTask->pMeta; SStreamMeta* pMeta = pStreamTask->pMeta;
STaskId hId = pStreamTask->hTaskInfo.id; STaskId hId = pStreamTask->hTaskInfo.id;
SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqWarn("s-task:0x%x failed to acquired it to exec step 2, scan wal quit", (int32_t) hId.taskId); tqWarn("s-task:0x%x failed to acquired it to exec step 2, scan wal quit", (int32_t) hId.taskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -874,7 +858,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to acquire stream task:0x%x during scan history data, task may have been destroyed", tqError("vgId:%d failed to acquire stream task:0x%x during scan history data, task may have been destroyed",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
@ -883,7 +868,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
// do recovery step1 // do recovery step1
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
char* pStatus = streamTaskGetStatus(pTask)->name; char* pStatus = streamTaskGetStatus(pTask).name;
// avoid multi-thread exec // avoid multi-thread exec
while (1) { while (1) {
@ -939,15 +924,15 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
if (retInfo.ret == TASK_SCANHISTORY_REXEC) { if (retInfo.ret == TASK_SCANHISTORY_REXEC) {
streamExecScanHistoryInFuture(pTask, retInfo.idleTime); streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else { } else {
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
ETaskStatus s = p->state; ETaskStatus s = p.state;
if (s == TASK_STATUS__PAUSE) { if (s == TASK_STATUS__PAUSE) {
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr, tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr,
el, pTask->execInfo.step1El, status); el, pTask->execInfo.step1El, status);
} else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) { } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) {
tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr,
p->name, pTask->execInfo.step1El); p.name, pTask->execInfo.step1El);
} }
} }
@ -961,7 +946,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
ASSERT(pTask->info.fillHistory == 1); ASSERT(pTask->info.fillHistory == 1);
// 1. get the related stream task // 1. get the related stream task
SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); SStreamTask* pStreamTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask == NULL) { if (pStreamTask == NULL) {
tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
pTask->streamTaskId.taskId, pTask->id.idStr); pTask->streamTaskId.taskId, pTask->id.idStr);
@ -1121,7 +1107,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64 tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64
" transId:%d it may have been destroyed", " transId:%d it may have been destroyed",
@ -1147,7 +1134,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
// todo save the checkpoint failed info // todo save the checkpoint failed info
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state; ETaskStatus status = streamTaskGetStatus(pTask).state;
if (req.mndTrigger == 1) { if (req.mndTrigger == 1) {
if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) { if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {

196
source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c
View File

@ -1,196 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
#include "tdbInt.h"
#include "tq.h"
// STqCheckInfoReader ========================================
struct STqCheckInfoReader {
STQ* pTq;
int64_t sver;
int64_t ever;
TBC* pCur;
};
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader) {
int32_t code = 0;
STqCheckInfoReader* pReader = NULL;
// alloc
pReader = (STqCheckInfoReader*)taosMemoryCalloc(1, sizeof(STqCheckInfoReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
// impl
code = tdbTbcOpen(pTq->pCheckStore, &pReader->pCur, NULL);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
code = tdbTbcMoveToFirst(pReader->pCur);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
tqInfo("vgId:%d, vnode checkinfo tq reader opened", TD_VID(pTq->pVnode));
*ppReader = pReader;
return code;
_err:
tqError("vgId:%d, vnode checkinfo tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppReader = NULL;
return code;
}
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader) {
int32_t code = 0;
tdbTbcClose((*ppReader)->pCur);
taosMemoryFree(*ppReader);
*ppReader = NULL;
return code;
}
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData) {
int32_t code = 0;
void* pKey = NULL;
void* pVal = NULL;
int32_t kLen = 0;
int32_t vLen = 0;
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
goto _exit;
}
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
pHdr->type = SNAP_DATA_TQ_CHECKINFO;
pHdr->size = vLen;
memcpy(pHdr->data, pVal, vLen);
_exit:
tdbFree(pKey);
tdbFree(pVal);
tqInfo("vgId:%d, vnode check info tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
return code;
_err:
tdbFree(pKey);
tdbFree(pVal);
tqError("vgId:%d, vnode check info tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
return code;
}
// STqCheckInfoWriter ========================================
struct STqCheckInfoWriter {
STQ* pTq;
int64_t sver;
int64_t ever;
TXN* txn;
};
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter) {
int32_t code = 0;
STqCheckInfoWriter* pWriter;
// alloc
pWriter = (STqCheckInfoWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pWriter->pTq = pTq;
pWriter->sver = sver;
pWriter->ever = ever;
if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
code = -1;
taosMemoryFree(pWriter);
goto _err;
}
*ppWriter = pWriter;
return code;
_err:
tqError("vgId:%d, tq check info writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback) {
int32_t code = 0;
STqCheckInfoWriter* pWriter = *ppWriter;
STQ* pTq = pWriter->pTq;
if (rollback) {
tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
} else {
code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err;
code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err;
}
taosMemoryFree(pWriter);
*ppWriter = NULL;
return code;
_err:
tqError("vgId:%d, tq check info writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqCheckInfo info = {0};
SDecoder decoder;
SDecoder* pDecoder = &decoder;
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = tDecodeSTqCheckInfo(pDecoder, &info);
if (code) goto _err;
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
if (code) goto _err;
code = tqMetaSaveCheckInfo(pTq, info.topic, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
if (code) goto _err;
tDecoderClear(pDecoder);
return code;
_err:
tDecoderClear(pDecoder);
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}

532
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -75,125 +75,98 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
return 0; return 0;
} }
int32_t tqMetaOpen(STQ* pTq) { int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) { SDecoder decoder = {0};
return -1;
}
if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) {
return -1;
}
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaClose(STQ* pTq) {
if (pTq->pExecStore) {
tdbTbClose(pTq->pExecStore);
}
if (pTq->pCheckStore) {
tdbTbClose(pTq->pCheckStore);
}
tdbClose(pTq->pMetaDB);
return 0;
}
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) {
return -1;
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) {
tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key);
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) {
return -1;
}
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
SDecoder decoder;
int32_t code = 0;
tdbTbcMoveToFirst(pCur);
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
STqCheckInfo info;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
code = tDecodeSTqCheckInfo(&decoder, &info); int32_t code = tDecodeSTqCheckInfo(&decoder, info);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; tDeleteSTqCheckInfo(info);
goto END; return TSDB_CODE_OUT_OF_MEMORY;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}
END:
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
return code; return code;
} }
int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){
SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqOffset(&decoder, info);
if (code != 0) {
tDeleteSTqOffset(info);
return TSDB_CODE_OUT_OF_MEMORY;
}
tDecoderClear(&decoder);
return code;
}
int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
}
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbDelete(ttb, key, kLen, txn));
TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
}
void* tqMetaGetOffset(STQ* pTq, const char* subkey){
void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey));
if (data == NULL) {
int vLen = 0;
if (tdbTbGet(pTq->pOffsetStore, subkey, strlen(subkey), &data, &vLen) < 0) {
tdbFree(data);
return NULL;
}
STqOffset offset = {0};
if (tqMetaDecodeOffsetInfo(&offset, data, vLen) != TDB_CODE_SUCCESS) {
tdbFree(data);
return NULL;
}
if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0){
tDeleteSTqOffset(&offset);
tdbFree(data);
return NULL;
}
tdbFree(data);
return taosHashGet(pTq->pOffset, subkey, strlen(subkey));
} else {
return data;
}
}
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
int32_t code; int32_t code = TDB_CODE_SUCCESS;
int32_t vlen; int32_t vlen;
void* buf = NULL; void* buf = NULL;
SEncoder encoder; SEncoder encoder;
tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
if (code < 0) { if (code < 0) {
goto end; goto END;
} }
tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 " epoch:%d vgId:%d", pHandle->subKey, tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 " epoch:%d vgId:%d", pHandle->subKey,
@ -202,71 +175,33 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
buf = taosMemoryCalloc(1, vlen); buf = taosMemoryCalloc(1, vlen);
if (buf == NULL) { if (buf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto END;
} }
tEncoderInit(&encoder, buf, vlen); tEncoderInit(&encoder, buf, vlen);
code = tEncodeSTqHandle(&encoder, pHandle); code = tEncodeSTqHandle(&encoder, pHandle);
if (code < 0) { if (code < 0) {
goto end; goto END;
} }
TXN* txn = NULL; TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pExecStore, key, (int)strlen(key), buf, vlen));
code = tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
if (code < 0) {
goto end;
}
code = tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn); END:
if (code < 0) {
goto end;
}
code = tdbCommit(pTq->pMetaDB, txn);
if (code < 0) {
goto end;
}
code = tdbPostCommit(pTq->pMetaDB, txn);
end:
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosMemoryFree(buf); taosMemoryFree(buf);
return code; return code;
} }
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
TXN* txn; int32_t code = TDB_CODE_SUCCESS;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) {
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
static int buildHandle(STQ* pTq, STqHandle* handle){
SVnode* pVnode = pTq->pVnode; SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
handle->pRef = walOpenRef(pVnode->pWal); handle->pRef = walOpenRef(pVnode->pWal);
if (handle->pRef == NULL) { if (handle->pRef == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
walSetRefVer(handle->pRef, handle->snapshotVer); TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer));
SReadHandle reader = { SReadHandle reader = {
.vnode = pVnode, .vnode = pVnode,
@ -282,18 +217,18 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId); qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId);
if (handle->execHandle.task == NULL) { if (handle->execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle->subKey); tqError("cannot create exec task for %s", handle->subKey);
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
void* scanner = NULL; void* scanner = NULL;
qExtractStreamScanner(handle->execHandle.task, &scanner); (void)qExtractStreamScanner(handle->execHandle.task, &scanner);
if (scanner == NULL) { if (scanner == NULL) {
tqError("cannot extract stream scanner for %s", handle->subKey); tqError("cannot extract stream scanner for %s", handle->subKey);
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner);
if (handle->execHandle.pTqReader == NULL) { if (handle->execHandle.pTqReader == NULL) {
tqError("cannot extract exec reader for %s", handle->subKey); tqError("cannot extract exec reader for %s", handle->subKey);
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
} else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0); handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0);
@ -308,7 +243,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) { if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) {
if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) { if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) {
tqError("nodesStringToNode error in sub stable, since %s", terrstr()); tqError("nodesStringToNode error in sub stable, since %s", terrstr());
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
} }
buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType, buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType,
@ -320,7 +255,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
if(ret != TDB_CODE_SUCCESS) { if(ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId); tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId);
taosArrayDestroy(tbUidList); taosArrayDestroy(tbUidList);
return -1; return TSDB_CODE_SCH_INTERNAL_ERROR;
} }
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid); tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
handle->execHandle.pTqReader = tqReaderOpen(pVnode); handle->execHandle.pTqReader = tqReaderOpen(pVnode);
@ -330,24 +265,23 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
return 0; return 0;
} }
static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){ static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder; SDecoder decoder = {0};
int32_t code = 0; int32_t code = TDB_CODE_SUCCESS;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
code = tDecodeSTqHandle(&decoder, handle); TQ_ERR_GO_TO_END(tDecodeSTqHandle(&decoder, handle));
if (code) goto end; TQ_ERR_GO_TO_END(tqMetaInitHandle(pTq, handle));
code = buildHandle(pTq, handle); tqInfo("tqMetaRestoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
if (code) goto end;
tqInfo("restoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
code = taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); code = taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
end: END:
tDecoderClear(&decoder); tDecoderClear(&decoder);
return code; return code;
} }
int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
@ -367,165 +301,189 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal);
if(buildHandle(pTq, handle) < 0){ if(tqMetaInitHandle(pTq, handle) < 0){
return -1; return -1;
} }
tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer); tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
} }
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* pExecStoreNew){ static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pExecStoreOld, &pCur, NULL) < 0) {
return -1;
}
TXN* txn;
if (tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
return -1;
}
void* pKey = NULL; void* pKey = NULL;
int kLen = 0; int kLen = 0;
void* pVal = NULL; void* pVal = NULL;
int vLen = 0; int vLen = 0;
TXN* txn = NULL;
tdbTbcMoveToFirst(pCur); int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
if (tdbTbUpsert(pExecStoreNew, pKey, kLen, pVal, vLen, txn) < 0) { TQ_ERR_GO_TO_END (tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn));
tqError("transform sub info error"); }
tdbFree(pKey);
tdbFree(pVal); TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn));
tdbTbcClose(pCur); TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn));
return -1;
} END:
} tdbFree(pKey);
tdbFree(pVal);
(void)tdbTbcClose(pCur);
return code;
}
int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) {
void* data = taosHashGet(pTq->pHandle, key, strlen(key));
if(data == NULL){
int vLen = 0;
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) {
tdbFree(data);
return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
}
STqHandle handle = {0};
if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){
tdbFree(data);
tqDestroyTqHandle(&handle);
return TSDB_CODE_OUT_OF_MEMORY;
}
tdbFree(data);
*pHandle = taosHashGet(pTq->pHandle, key, strlen(key));
if(*pHandle == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
*pHandle = data;
}
return TDB_CODE_SUCCESS;
}
int32_t tqMetaOpenTdb(STQ* pTq) {
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.offset.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pOffsetStore, 0));
END:
return code;
}
static int32_t replaceTqPath(char** path){
char* tpath = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME));
taosMemoryFree(*path);
*path = tpath;
return TDB_CODE_SUCCESS;
}
static int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
int32_t code = 0;
STqCheckInfo info = {0};
TQ_ERR_GO_TO_END(tdbTbcOpen(pTq->pCheckStore, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen));
TQ_ERR_GO_TO_END(taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)));
}
info.colIdList = NULL;
END:
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
tdbTbcClose(pCur); tdbTbcClose(pCur);
tDeleteSTqCheckInfo(&info);
return code;
}
if (tdbCommit(pMetaDB, txn) < 0) { int32_t tqMetaOpen(STQ* pTq) {
return -1; char* maindb = NULL;
char* offsetNew = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){
TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{
TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
(void)taosRemoveFile(maindb);
} }
if (tdbPostCommit(pMetaDB, txn) < 0) { TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
return -1; if(taosCheckExistFile(offsetNew)){
TQ_ERR_GO_TO_END(tqOffsetRestoreFromFile(pTq, offsetNew));
(void)taosRemoveFile(offsetNew);
} }
return 0;
TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq));
END:
taosMemoryFree(maindb);
taosMemoryFree(offsetNew);
return code;
} }
int32_t tqMetaTransform(STQ* pTq) { int32_t tqMetaTransform(STQ* pTq) {
int32_t len = strlen(pTq->path) + 64; int32_t code = TDB_CODE_SUCCESS;
char* maindb = taosMemoryCalloc(1, len);
sprintf(maindb, "%s%s%s", pTq->path, TD_DIRSEP, TDB_MAINDB_NAME);
if(!taosCheckExistFile(maindb)){
taosMemoryFree(maindb);
char* tpath = taosMemoryCalloc(1, len);
if(tpath == NULL){
return -1;
}
sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe");
taosMemoryFree(pTq->path);
pTq->path = tpath;
return tqMetaOpen(pTq);
}
int32_t code = 0;
TDB* pMetaDB = NULL; TDB* pMetaDB = NULL;
TTB* pExecStore = NULL; TTB* pExecStore = NULL;
TTB* pCheckStore = NULL; TTB* pCheckStore = NULL;
char* offsetNew = NULL; char* offsetNew = NULL;
char* offset = tqOffsetBuildFName(pTq->path, 0); char* offset = NULL;
if(offset == NULL){ TQ_ERR_GO_TO_END(tqBuildFName(&offset, pTq->path, TQ_OFFSET_NAME));
code = -1;
goto END;
}
TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0));
if (tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL) < 0) { TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
code = -1; TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
goto END;
}
if (tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0) < 0) { TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore));
code = -1; TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
goto END;
}
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0) < 0) { TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
code = -1; if(taosCheckExistFile(offset)) {
goto END; if (taosCopyFile(offset, offsetNew) < 0) {
}
char* tpath = taosMemoryCalloc(1, len);
if(tpath == NULL){
code = -1;
goto END;
}
sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe");
taosMemoryFree(pTq->path);
pTq->path = tpath;
if (tqMetaOpen(pTq) < 0) {
code = -1;
goto END;
}
if( tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore) < 0){
code = -1;
goto END;
}
if(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore) < 0){
code = -1;
goto END;
}
tdbTbClose(pExecStore);
pExecStore = NULL;
tdbTbClose(pCheckStore);
pCheckStore = NULL;
tdbClose(pMetaDB);
pMetaDB = NULL;
offsetNew = tqOffsetBuildFName(pTq->path, 0);
if(offsetNew == NULL){
code = -1;
goto END;
}
if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){
tqError("copy offset file error"); tqError("copy offset file error");
code = -1; } else {
goto END; (void)taosRemoveFile(offset);
}
} }
taosRemoveFile(maindb); END:
taosRemoveFile(offset);
END:
taosMemoryFree(maindb);
taosMemoryFree(offset); taosMemoryFree(offset);
taosMemoryFree(offsetNew); taosMemoryFree(offsetNew);
tdbTbClose(pExecStore); //return 0 always, so ignore
tdbTbClose(pCheckStore); (void)tdbTbClose(pExecStore);
tdbClose(pMetaDB); (void)tdbTbClose(pCheckStore);
(void)tdbClose(pMetaDB);
return code; return code;
} }
int32_t tqMetaGetHandle(STQ* pTq, const char* key) { int32_t tqMetaClose(STQ* pTq) {
void* pVal = NULL; if (pTq->pExecStore) {
int vLen = 0; (void)tdbTbClose(pTq->pExecStore);
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &pVal, &vLen) < 0) {
return -1;
} }
STqHandle handle = {0}; if (pTq->pCheckStore) {
int code = restoreHandle(pTq, pVal, vLen, &handle); (void)tdbTbClose(pTq->pCheckStore);
if (code < 0){
tqDestroyTqHandle(&handle);
} }
tdbFree(pVal); if (pTq->pOffsetStore) {
return code; (void)tdbTbClose(pTq->pOffsetStore);
}
(void)tdbClose(pTq->pMetaDB);
return 0;
} }

213
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -16,192 +16,71 @@
#include "tq.h" #include "tq.h"
struct STqOffsetStore { int32_t tqBuildFName(char** data, const char* path, char* name) {
STQ* pTq; int32_t len = strlen(path) + strlen(name) + 2;
SHashObj* pHash; // SHashObj<subscribeKey, offset> char* fname = taosMemoryCalloc(1, len);
int8_t needCommit;
};
char* tqOffsetBuildFName(const char* path, int32_t fVer) {
int32_t len = strlen(path);
char* fname = taosMemoryCalloc(1, len + 40);
if(fname == NULL) { if(fname == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
snprintf(fname, len + 40, "%s/offset-ver%d", path, fVer); int32_t code = snprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name);
return fname; if (code < 0){
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(fname);
return code;
}
*data = fname;
return TDB_CODE_SUCCESS;
} }
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); int32_t code = TDB_CODE_SUCCESS;
void* pMemBuf = NULL;
TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
return TSDB_CODE_SUCCESS; code = TDB_CODE_SUCCESS;
goto END;
} }
int32_t vgId = TD_VID(pStore->pTq->pVnode); int64_t ret = 0;
int64_t code = 0;
int32_t size = 0; int32_t size = 0;
while (1) { while (1) {
if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if ((ret = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) {
if (code == 0) { if (ret != 0) {
break; code = TSDB_CODE_INVALID_MSG;
} else {
return -1;
} }
goto END;
} }
size = htonl(size); size = htonl(size);
void* pMemBuf = taosMemoryCalloc(1, size); pMemBuf = taosMemoryCalloc(1, size);
if (pMemBuf == NULL) { if (pMemBuf == NULL) {
tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); code = TSDB_CODE_OUT_OF_MEMORY;
terrno = TSDB_CODE_OUT_OF_MEMORY; goto END;
return -1;
} }
if ((code = taosReadFile(pFile, pMemBuf, size)) != size) { if (taosReadFile(pFile, pMemBuf, size) != size) {
taosMemoryFree(pMemBuf); terrno = TSDB_CODE_INVALID_MSG;
return -1; goto END;
} }
STqOffset offset = {0}; STqOffset offset = {0};
SDecoder decoder; TQ_ERR_GO_TO_END(tqMetaDecodeOffsetInfo(&offset, pMemBuf, size));
tDecoderInit(&decoder, pMemBuf, size); code = taosHashPut(pTq->pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset));
if (tDecodeSTqOffset(&decoder, &offset) < 0) { if (code != TDB_CODE_SUCCESS) {
tDeleteSTqOffset(&offset);
goto END;
}
TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size));
tqInfo("tq: offset restore from file to tdb, subkey:%s", offset.subKey);
taosMemoryFree(pMemBuf); taosMemoryFree(pMemBuf);
tDecoderClear(&decoder); pMemBuf = NULL;
}
END:
taosCloseFile(&pFile);
taosMemoryFree(pMemBuf);
return code; return code;
}
tDecoderClear(&decoder);
if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) {
return -1;
}
// todo remove this
if (offset.val.type == TMQ_OFFSET__LOG) {
STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey));
if (pHandle) {
if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) {
// tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, pHandle->subKey,
// offset.val.version);
}
}
}
taosMemoryFree(pMemBuf);
}
taosCloseFile(&pFile);
return TSDB_CODE_SUCCESS;
}
STqOffsetStore* tqOffsetOpen(STQ* pTq) {
STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore));
if (pStore == NULL) {
return NULL;
}
pStore->pTq = pTq;
pStore->needCommit = 0;
pTq->pOffsetStore = pStore;
pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (pStore->pHash == NULL) {
taosMemoryFree(pStore);
return NULL;
}
taosHashSetFreeFp(pStore->pHash, tOffsetDestroy);
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
if (tqOffsetRestoreFromFile(pStore, fname) < 0) {
taosMemoryFree(fname);
taosMemoryFree(pStore);
return NULL;
}
taosMemoryFree(fname);
return pStore;
}
void tqOffsetClose(STqOffsetStore* pStore) {
if(pStore == NULL) return;
tqOffsetCommitFile(pStore);
taosHashCleanup(pStore->pHash);
taosMemoryFree(pStore);
}
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) {
return (STqOffset*)taosHashGet(pStore->pHash, subscribeKey, strlen(subscribeKey));
}
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) {
pStore->needCommit = 1;
return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
}
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) {
return taosHashRemove(pStore->pHash, subscribeKey, strlen(subscribeKey));
}
int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
if (!pStore->needCommit) {
return 0;
}
// TODO file name should be with a newer version
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
const char* err = strerror(errno);
tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pStore->pHash, pIter);
if (pIter == NULL) {
break;
}
STqOffset* pOffset = (STqOffset*)pIter;
int32_t bodyLen;
int32_t code;
tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code);
if (code < 0) {
taosHashCancelIterate(pStore->pHash, pIter);
return -1;
}
int32_t totLen = INT_BYTES + bodyLen;
void* buf = taosMemoryCalloc(1, totLen);
void* abuf = POINTER_SHIFT(buf, INT_BYTES);
*(int32_t*)buf = htonl(bodyLen);
SEncoder encoder;
tEncoderInit(&encoder, abuf, bodyLen);
tEncodeSTqOffset(&encoder, pOffset);
// write file
int64_t writeLen;
if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
taosHashCancelIterate(pStore->pHash, pIter);
taosMemoryFree(buf);
return -1;
}
taosMemoryFree(buf);
}
// close and rename file
taosCloseFile(&pFile);
pStore->needCommit = 0;
return 0;
} }

168
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -1,168 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
#include "tdbInt.h"
#include "tq.h"
// STqOffsetReader ========================================
struct STqOffsetReader {
STQ* pTq;
int64_t sver;
int64_t ever;
int8_t readEnd;
};
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader) {
STqOffsetReader* pReader = NULL;
pReader = taosMemoryCalloc(1, sizeof(STqOffsetReader));
if (pReader == NULL) {
*ppReader = NULL;
return -1;
}
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
tqInfo("vgId:%d, vnode snapshot tq offset reader opened", TD_VID(pTq->pVnode));
*ppReader = pReader;
return 0;
}
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader) {
taosMemoryFree(*ppReader);
*ppReader = NULL;
return 0;
}
int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
if (pReader->readEnd != 0) return 0;
char* fname = tqOffsetBuildFName(pReader->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
taosMemoryFree(fname);
return 0;
}
int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
taosCloseFile(&pFile);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);
SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr));
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
return terrno;
}
void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr));
int64_t contLen = taosReadFile(pFile, abuf, sz);
if (contLen != sz) {
taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1;
}
buf->size = sz;
buf->type = SNAP_DATA_TQ_OFFSET;
*ppData = (uint8_t*)buf;
pReader->readEnd = 1;
taosCloseFile(&pFile);
return 0;
}
// STqOffseWriter ========================================
struct STqOffsetWriter {
STQ* pTq;
int64_t sver;
int64_t ever;
int32_t tmpFileVer;
char* fname;
};
int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter) {
int32_t code = 0;
STqOffsetWriter* pWriter;
pWriter = (STqOffsetWriter*)taosMemoryCalloc(1, sizeof(STqOffsetWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pWriter->pTq = pTq;
pWriter->sver = sver;
pWriter->ever = ever;
*ppWriter = pWriter;
return code;
_err:
tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) {
STqOffsetWriter* pWriter = *ppWriter;
STQ* pTq = pWriter->pTq;
char* fname = tqOffsetBuildFName(pTq->path, 0);
if (rollback) {
if (taosRemoveFile(pWriter->fname) < 0) {
taosMemoryFree(fname);
return -1;
}
} else {
if (taosRenameFile(pWriter->fname, fname) < 0) {
taosMemoryFree(fname);
return -1;
}
if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) {
taosMemoryFree(fname);
return -1;
}
}
taosMemoryFree(fname);
taosMemoryFree(pWriter->fname);
taosMemoryFree(pWriter);
*ppWriter = NULL;
return 0;
}
int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData) {
STQ* pTq = pWriter->pTq;
pWriter->tmpFileVer = 1;
pWriter->fname = tqOffsetBuildFName(pTq->path, pWriter->tmpFileVer);
TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE);
SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
int64_t size = pHdr->size;
if (pFile) {
int64_t contLen = taosWriteFile(pFile, pHdr->data, size);
if (contLen != size) {
taosCloseFile(&pFile);
return -1;
}
taosCloseFile(&pFile);
} else {
return -1;
}
return 0;
}

6
source/dnode/vnode/src/tq/tqRead.c
View File

@ -341,10 +341,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
memcpy(data, pBody, len); memcpy(data, pBody, len);
SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data};
*pItem = (SStreamQueueItem*)streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT); code = streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT, (SStreamDataSubmit**)pItem);
if (*pItem == NULL) { if (code != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
terrno = code;
tqError("%s failed to create data submit for stream since out of memory", id); tqError("%s failed to create data submit for stream since out of memory", id);
return code; return code;
} }

62
source/dnode/vnode/src/tq/tqHandleSnapshot.c → source/dnode/vnode/src/tq/tqSnapshot.c
View File

@ -23,9 +23,10 @@ struct STqSnapReader {
int64_t sver; int64_t sver;
int64_t ever; int64_t ever;
TBC* pCur; TBC* pCur;
int8_t type;
}; };
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader) { int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) {
int32_t code = 0; int32_t code = 0;
STqSnapReader* pReader = NULL; STqSnapReader* pReader = NULL;
@ -38,9 +39,21 @@ int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** p
pReader->pTq = pTq; pReader->pTq = pTq;
pReader->sver = sver; pReader->sver = sver;
pReader->ever = ever; pReader->ever = ever;
pReader->type = type;
// impl // impl
code = tdbTbcOpen(pTq->pExecStore, &pReader->pCur, NULL); TTB *pTb = NULL;
if (type == SNAP_DATA_TQ_CHECKINFO) {
pTb = pTq->pCheckStore;
} else if (type == SNAP_DATA_TQ_HANDLE) {
pTb = pTq->pExecStore;
} else if (type == SNAP_DATA_TQ_OFFSET) {
pTb = pTq->pOffsetStore;
} else {
code = TSDB_CODE_INVALID_MSG;
goto _err;
}
code = tdbTbcOpen(pTb, &pReader->pCur, NULL);
if (code) { if (code) {
taosMemoryFree(pReader); taosMemoryFree(pReader);
goto _err; goto _err;
@ -91,7 +104,7 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
} }
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
pHdr->type = SNAP_DATA_TQ_HANDLE; pHdr->type = pReader->type;
pHdr->size = vLen; pHdr->size = vLen;
memcpy(pHdr->data, pVal, vLen); memcpy(pHdr->data, pVal, vLen);
@ -169,7 +182,7 @@ _err:
return code; return code;
} }
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
STQ* pTq = pWriter->pTq; STQ* pTq = pWriter->pTq;
SDecoder decoder = {0}; SDecoder decoder = {0};
@ -180,7 +193,7 @@ int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
code = tDecodeSTqHandle(pDecoder, &handle); code = tDecodeSTqHandle(pDecoder, &handle);
if (code) goto end; if (code) goto end;
taosWLockLatch(&pTq->lock); taosWLockLatch(&pTq->lock);
code = tqMetaSaveHandle(pTq, handle.subKey, &handle); code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
end: end:
@ -189,3 +202,42 @@ end:
tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code); tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code);
return code; return code;
} }
int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqCheckInfo info = {0};
if(tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){
goto _err;
}
code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
tDeleteSTqCheckInfo(&info);
if (code) goto _err;
return code;
_err:
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}
int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqOffset info = {0};
if(tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){
goto _err;
}
code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
tDeleteSTqOffset(&info);
if (code) goto _err;
return code;
_err:
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}

20
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -226,15 +226,15 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
} }
// not in ready state, do not handle the data from wal // not in ready state, do not handle the data from wal
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState->name); tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState.name);
return false; return false;
} }
// fill-history task has entered into the last phase, no need to anything // fill-history task has entered into the last phase, no need to anything
if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
ASSERT(pState->state == TASK_STATUS__READY); ASSERT(pState.state == TASK_STATUS__READY);
// the maximum version of data in the WAL has reached already, the step2 is done // the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer); pTask->dataRange.range.maxVer);
@ -324,7 +324,8 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
STaskId* pTaskId = taosArrayGet(pTaskList, i); STaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId); SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
continue; continue;
} }
@ -337,7 +338,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = false; *pScanIdle = false;
// seek the stored version and extract data from WAL // seek the stored version and extract data from WAL
int32_t code = setWalReaderStartOffset(pTask, vgId); code = setWalReaderStartOffset(pTask, vgId);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
@ -348,9 +349,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState->name); tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
@ -364,6 +365,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
code = streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
taosArrayDestroy(pTaskList);
return -1; return -1;
} }
} }

Some files were not shown because too many files have changed in this diff Show More