xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: return error code

This commit is contained in:
kailixu 2024-07-22 15:53:20 +08:00
parent b337fa628b 5328c6dbea
commit cdc85d9588
307 changed files with 17001 additions and 12625 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Jenkinsfile2
View File

@ -401,7 +401,7 @@ pipeline {
}
}
stage('linux test') {
agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() }
when {
changeRequest()
@ -416,7 +416,7 @@ pipeline {
echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
}
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 150, unit: 'MINUTES'){
timeout(time: 200, unit: 'MINUTES'){
pre_test()
script {
sh '''
@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2
date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + '''
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 1200 ''' + extra_param + '''
'''
}
}

2
docs/en/12-taos-sql/12-distinguished.md
View File

@ -38,7 +38,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## Windowed Queries
Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are four kinds of windows: time window, status window, session window, and event window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows:
Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are five kinds of windows: time window, status window, session window, event window, and count window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows:
```sql
window_clause: {

2
docs/zh/12-taos-sql/12-distinguished.md
View File

@ -39,7 +39,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## 窗口切分查询
TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）四种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。
TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）、计数窗口（count window）五种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。
窗口子句语法如下：

2
include/common/rsync.h
View File

@ -12,7 +12,7 @@ extern "C" {
#include "tarray.h"
void stopRsync();
void startRsync();
int32_t startRsync();
int32_t uploadByRsync(const char* id, const char* path);
int32_t downloadRsync(const char* id, const char* path);
int32_t deleteRsync(const char* id);

2
include/common/tdatablock.h
View File

@ -253,7 +253,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);
SSDataBlock* createDataBlock();
void* blockDataDestroy(SSDataBlock* pBlock);
void blockDataDestroy(SSDataBlock* pBlock);
void blockDataFreeRes(SSDataBlock* pBlock);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createSpecialDataBlock(EStreamType type);

1
include/common/tmsg.h
View File

@ -3612,6 +3612,7 @@ typedef struct {
int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset);
int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset);
void tDeleteSTqOffset(void* val);
typedef struct SMqVgOffset {
int64_t consumerId;

102
include/libs/executor/storageapi.h
View File

@ -78,7 +78,7 @@ typedef struct SMetaEntry {
} smaEntry;
};
uint8_t* pBuf;
uint8_t* pBuf;
SColCmprWrapper colCmpr; // col compress alg
} SMetaEntry;
@ -105,15 +105,15 @@ typedef struct SMTbCursor {
} SMTbCursor;
typedef struct SMCtbCursor {
struct SMeta* pMeta;
void* pCur;
tb_uid_t suid;
void* pKey;
void* pVal;
int kLen;
int vLen;
int8_t paused;
int lock;
struct SMeta* pMeta;
void* pCur;
tb_uid_t suid;
void* pKey;
void* pVal;
int kLen;
int vLen;
int8_t paused;
int lock;
} SMCtbCursor;
typedef struct SRowBuffPos {
@ -135,17 +135,17 @@ typedef struct SMetaTableInfo {
typedef struct SSnapContext {
struct SMeta* pMeta;
int64_t snapVersion;
void* pCur;
int64_t suid;
int8_t subType;
SHashObj* idVersion;
SHashObj* suidInfo;
SArray* idList;
int32_t index;
int8_t withMeta;
int8_t queryMeta; // true-get meta, false-get data
bool hasPrimaryKey;
int64_t snapVersion;
void* pCur;
int64_t suid;
int8_t subType;
SHashObj* idVersion;
SHashObj* suidInfo;
SArray* idList;
int32_t index;
int8_t withMeta;
int8_t queryMeta; // true-get meta, false-get data
bool hasPrimaryKey;
} SSnapContext;
typedef struct {
@ -229,15 +229,15 @@ typedef struct SStoreTqReader {
bool (*tqReaderCurrentBlockConsumed)();
struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it
// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
// int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it
int32_t (*tqReaderSetSubmitMsg)(); // todo remove it
// bool (*tqReaderNextBlockFilterOut)();
// bool (*tqReaderNextBlockFilterOut)();
} SStoreTqReader;
typedef struct SStoreSnapshotFn {
bool (*taosXGetTablePrimaryKey)(SSnapContext *ctx);
void (*taosXSetTablePrimaryKey)(SSnapContext *ctx, int64_t uid);
bool (*taosXGetTablePrimaryKey)(SSnapContext* ctx);
void (*taosXSetTablePrimaryKey)(SSnapContext* ctx, int64_t uid);
int32_t (*setForSnapShot)(SSnapContext* ctx, int64_t uid);
int32_t (*destroySnapshot)(SSnapContext* ctx);
SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx);
@ -270,7 +270,7 @@ typedef struct SStoreMeta {
int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
int32_t payloadLen, double selectivityRatio);
void* (*storeGetIndexInfo)(void *pVnode);
void* (*storeGetIndexInfo)(void* pVnode);
void* (*getInvertIndex)(void* pVnode);
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
@ -329,29 +329,31 @@ typedef struct {
typedef struct SStateStore {
int32_t (*streamStatePutParName)(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache);
int32_t (*streamStateGetParName)(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache,
int32_t* pWinCode);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
int32_t (*streamStateClearBuff)(SStreamState* pState, void* pVal);
int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void (*streamStateReleaseBuf)(SStreamState* pState, void* pVal, bool used);
void (*streamStateClearBuff)(SStreamState* pState, void* pVal);
void (*streamStateFreeVal)(void* val);
int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal);
int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateClear)(SStreamState* pState);
void (*streamStateDel)(SStreamState* pState, const SWinKey* key);
void (*streamStateClear)(SStreamState* pState);
void (*streamStateSetNumber)(SStreamState* pState, int32_t number, int32_t tsIdex);
int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
void (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
int32_t (*streamStateFillPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t (*streamStateFillGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
void (*streamStateFillDel)(SStreamState* pState, const SWinKey* key);
int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur);
int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur);
void (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur);
void (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur);
SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key);
SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key);
@ -363,26 +365,29 @@ typedef struct SStateStore {
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
int32_t* pVLen);
int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
int32_t (*streamStateSessionReset)(SStreamState* pState, void* pVal);
int32_t (*streamStateSessionClear)(SStreamState* pState);
int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key);
void (*streamStateSessionReset)(SStreamState* pState, void* pVal);
void (*streamStateSessionClear)(SStreamState* pState);
int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateCountGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen);
void** ppVal, int32_t* pVLen, int32_t* pWinCode);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType,
int32_t pkLen, SUpdateInfo** ppInfo);
int32_t (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol,
int32_t primaryKeyCol, TSKEY* pMaxResTs);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
@ -391,7 +396,8 @@ typedef struct SStateStore {
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
@ -413,7 +419,7 @@ typedef struct SStateStore {
SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState);
void (*streamStateCommit)(SStreamState* pState);
void (*streamStateDestroy)(SStreamState* pState, bool remove);
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);

2
include/libs/function/function.h
View File

@ -294,7 +294,7 @@ typedef struct SPoint {
void *val;
} SPoint;
int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
void taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
int32_t inputType);
#define LEASTSQUARES_DOUBLE_ITEM_LENGTH 25

3
include/libs/qcom/query.h
View File

@ -192,6 +192,7 @@ typedef struct SBoundColInfo {
int16_t* pColIndex; // bound index => schema index
int32_t numOfCols;
int32_t numOfBound;
bool hasBoundCols;
} SBoundColInfo;
typedef struct STableColsData {
@ -334,7 +335,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
char* parseTagDatatoJson(void* p);
void parseTagDatatoJson(void* p, char** jsonStr);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);

5
include/libs/scalar/filter.h
View File

@ -58,12 +58,13 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict
extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows);
extern int32_t filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows,
bool *keep);
/* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,
SNode **pOtherCond);
bool filterIsMultiTableColsCond(SNode *pCond);
int32_t filterIsMultiTableColsCond(SNode *pCond, bool *res);
EConditionType filterClassifyCondition(SNode *pNode);
#ifdef __cplusplus

40
include/libs/stream/streamState.h
View File

@ -32,7 +32,7 @@ extern "C" {
SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId);
void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState);
void streamStateCommit(SStreamState* pState);
void streamStateDestroy(SStreamState* pState, bool remove);
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
int32_t streamStateDelTaskDb(SStreamState* pState);
@ -41,22 +41,23 @@ int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void*
int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
bool streamStateCheck(SStreamState* pState, const SWinKey* key);
int32_t streamStateGetByPos(SStreamState* pState, void* pos, void** pVal);
int32_t streamStateDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateClear(SStreamState* pState);
void streamStateDel(SStreamState* pState, const SWinKey* key);
void streamStateClear(SStreamState* pState);
void streamStateSetNumber(SStreamState* pState, int32_t number, int32_t tsIdex);
int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
void streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
// session window
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key);
int32_t streamStateSessionReset(SStreamState* pState, void* pVal);
int32_t streamStateSessionClear(SStreamState* pState);
int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen, int32_t* pWinCode);
void streamStateSessionDel(SStreamState* pState, const SSessionKey* key);
void streamStateSessionReset(SStreamState* pState, void* pVal);
void streamStateSessionClear(SStreamState* pState);
int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
int32_t streamStateCountGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey);
@ -70,21 +71,22 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons
// state window
int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
// fill
int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key);
void streamStateFillDel(SStreamState* pState, const SWinKey* key);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
int32_t streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
int32_t streamStateClearBuff(SStreamState* pState, void* pVal);
int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void streamStateReleaseBuf(SStreamState* pState, void* pVal, bool used);
void streamStateClearBuff(SStreamState* pState, void* pVal);
void streamStateFreeVal(void* val);
// count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen);
int32_t* pVLen, int32_t* pWinCode);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
@ -97,11 +99,11 @@ void streamStateResetCur(SStreamStateCur* pCur);
int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
void streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal, bool onlyCache, int32_t* pWinCode);
void streamStateReloadInfo(SStreamState* pState, TSKEY ts);

116
include/libs/stream/tstream.h
View File

@ -59,8 +59,6 @@ extern "C" {
#define STREAM_EXEC_T_STOP_ALL_TASKS (-5)
#define STREAM_EXEC_T_RESUME_TASK (-6)
#define STREAM_EXEC_T_ADD_FAILED_TASK (-7)
// the load and start stream task should be executed after snode has started successfully, since the load of stream
// tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources.
typedef struct SStreamTask SStreamTask;
typedef struct SStreamQueue SStreamQueue;
@ -190,8 +188,8 @@ typedef struct {
SSDataBlock* pBlock;
} SStreamRefDataBlock;
SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type);
void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit);
void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
typedef struct {
char* qmsg;
@ -530,13 +528,13 @@ typedef struct STaskUpdateEntry {
typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5);
void tFreeStreamTask(SStreamTask* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status);
int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5, SStreamTask** pTask);
void tFreeStreamTask(SStreamTask* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@ -631,8 +629,10 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo);
#if 0
SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId);
#endif
void streamTaskInputFail(SStreamTask* pTask);
@ -641,8 +641,8 @@ bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask);
int32_t createStreamTaskIdStr(int64_t streamId, int32_t taskId, const char** pId);
SStreamTaskState streamTaskGetStatus(const SStreamTask* pTask);
const char* streamTaskGetStatusStr(ETaskStatus status);
void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask);
@ -659,10 +659,10 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
int32_t streamExecTask(SStreamTask* pTask);
int32_t streamResumeTask(SStreamTask* pTask);
void streamResumeTask(SStreamTask* pTask);
int32_t streamTrySchedExec(SStreamTask* pTask);
int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int32_t taskId, int32_t execType);
int32_t streamTaskResumeInFuture(SStreamTask* pTask);
void streamTaskResumeInFuture(SStreamTask* pTask);
void streamTaskClearSchedIdleInfo(SStreamTask* pTask);
void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime);
@ -675,8 +675,8 @@ int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTa
int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp);
// check downstream status
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
int32_t streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id);
void streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
void streamTaskStopMonitorCheckRsp(STaskCheckInfo* pInfo, const char* id);
void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo);
// fill-history task
@ -724,43 +724,43 @@ int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* p
SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta
void streamMetaInit();
void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn,
int32_t vgId, int64_t stage, startComplete_fn_t fn);
void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
SStreamTask* streamMetaAcquireOneTask(SStreamTask* pTask);
void streamMetaClear(SStreamMeta* pMeta);
void streamMetaInitBackend(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta);
int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
void streamMetaNotifyClose(SStreamMeta* pMeta);
void streamMetaStartHb(SStreamMeta* pMeta);
bool streamMetaTaskInTimer(SStreamMeta* pMeta);
int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
int64_t endTs, bool ready);
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta);
int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaInit();
void streamMetaCleanup();
int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, int32_t vgId,
int64_t stage, startComplete_fn_t fn, SStreamMeta** pMeta);
void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
int32_t streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
int32_t streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, SStreamTask** pTask);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
void streamMetaAcquireOneTask(SStreamTask* pTask);
void streamMetaClear(SStreamMeta* pMeta);
void streamMetaInitBackend(SStreamMeta* pMeta);
int32_t streamMetaCommit(SStreamMeta* pMeta);
int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
void streamMetaNotifyClose(SStreamMeta* pMeta);
void streamMetaStartHb(SStreamMeta* pMeta);
bool streamMetaTaskInTimer(SStreamMeta* pMeta);
int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
int64_t endTs, bool ready);
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta);
int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta);
void streamMetaWUnLock(SStreamMeta* pMeta);
void streamMetaResetStartInfo(STaskStartInfo* pMeta, int32_t vgId);
SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta);
int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pTaskList);
void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader);
void streamMetaLoadAllTasks(SStreamMeta* pMeta);
int32_t streamMetaStartAllTasks(SStreamMeta* pMeta);
@ -771,6 +771,8 @@ int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer
tmr_h streamTimerGetInstance();
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg);
// checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
@ -787,14 +789,14 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp
int32_t setCode);
int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SVUpdateCheckpointInfoReq* pReq);
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();
int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes);
// stream task state machine, and event handling
SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
void* streamDestroyStateMachine(SStreamTaskSM* pSM);
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn,
void* param);
int32_t streamCreateStateMachine(SStreamTask* pTask);
void streamDestroyStateMachine(SStreamTaskSM* pSM);
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn,
void* param);
int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn,
void* param);
int32_t streamTaskRestoreStatus(SStreamTask* pTask);
@ -805,7 +807,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendPreparedCheckpointsourceRsp(SStreamTask* pTask);
int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
#ifdef __cplusplus

67
include/libs/stream/tstreamFileState.h
View File

@ -18,10 +18,10 @@
#include "os.h"
#include "storageapi.h"
#include "tarray.h"
#include "tdef.h"
#include "tlist.h"
#include "storageapi.h"
#ifdef __cplusplus
extern "C" {
@ -30,62 +30,66 @@ extern "C" {
typedef struct SStreamFileState SStreamFileState;
typedef SList SStreamSnapshot;
typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef void* (*_state_buff_get_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef int32_t (*_state_buff_remove_fn)(void* pRowBuff, const void* pKey, size_t keyLen);
typedef int32_t (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos);
typedef void (*_state_buff_cleanup_fn)(void* pRowBuff);
typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num);
typedef void (*_state_buff_remove_by_pos_fn)(SStreamFileState* pState, SRowBuffPos* pPos);
typedef void (*_state_buff_cleanup_fn)(void* pRowBuff);
typedef void* (*_state_buff_create_statekey_fn)(SRowBuffPos* pPos, int64_t num);
typedef int32_t (*_state_file_remove_fn)(SStreamFileState* pFileState, const void* pKey);
typedef int32_t (*_state_file_get_fn)(SStreamFileState* pFileState, void* pKey, void* data, int32_t* pDataLen);
typedef int32_t (*_state_file_clear_fn)(SStreamState* pState);
typedef int32_t (*_state_fun_get_fn) (SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
typedef int32_t (*_state_fun_get_fn)(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
typedef int32_t (*range_cmpr_fn)(const SSessionKey* pWin1, const SSessionKey* pWin2);
SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId,
int64_t checkpointId, int8_t type);
GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId,
int8_t type);
void streamFileStateDestroy(SStreamFileState* pFileState);
void streamFileStateClear(SStreamFileState* pFileState);
bool needClearDiskBuff(SStreamFileState* pFileState);
void streamFileStateReleaseBuff(SStreamFileState* pFileState, SRowBuffPos* pPos, bool used);
int32_t streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
void streamFileStateClearBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
int32_t deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
int32_t getRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
void deleteRowBuff(SStreamFileState* pFileState, const void* pKey, int32_t keyLen);
int32_t getRowBuffByPos(SStreamFileState* pFileState, SRowBuffPos* pPos, void** pVal);
bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen);
void putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t putFreeBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
void recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list);
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark);
int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState);
void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts);
void* getRowStateBuff(SStreamFileState* pFileState);
void* getStateFileStore(SStreamFileState* pFileState);
bool isDeteled(SStreamFileState* pFileState, TSKEY ts);
bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap);
void* getRowStateBuff(SStreamFileState* pFileState);
void* getStateFileStore(SStreamFileState* pFileState);
bool isDeteled(SStreamFileState* pFileState, TSKEY ts);
bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap);
SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState);
int32_t getRowStateRowSize(SStreamFileState* pFileState);
// session window
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen);
int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen);
int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos);
void deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos);
int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur,
const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen);
SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId);
void recoverSesssion(SStreamFileState* pFileState, int64_t ckId);
void sessionWinStateClear(SStreamFileState* pFileState);
void sessionWinStateCleanup(void* pBuff);
@ -94,20 +98,23 @@ SStreamStateCur* sessionWinStateSeekKeyCurrentPrev(SStreamFileState* pFileState,
SStreamStateCur* sessionWinStateSeekKeyCurrentNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* sessionWinStateSeekKeyNext(SStreamFileState* pFileState, const SSessionKey* pWinKey);
SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SSessionKey* pWinKey, COUNT_TYPE count);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey, range_cmpr_fn cmpFn);
int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen);
void sessionWinStateMoveToNext(SStreamStateCur* pCur);
int32_t sessionWinStateGetKeyByRange(SStreamFileState* pFileState, const SSessionKey* key, SSessionKey* curKey,
range_cmpr_fn cmpFn);
// state window
int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
state_key_cmpr_fn fn, void** pVal, int32_t* pVLen, int32_t* pWinCode);
// count window
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen);
int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal,
int32_t* pVLen, int32_t* pWinCode);
int32_t createCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
//function
int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
// function
int32_t getSessionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen,
int32_t* pWinCode);
int32_t getFunctionRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen, void** pVal, int32_t* pVLen);
#ifdef __cplusplus

9
include/libs/stream/tstreamUpdate.h
View File

@ -25,9 +25,12 @@
extern "C" {
#endif
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen,
SUpdateInfo** ppInfo);
int32_t updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol,
TSKEY* pMaxResTs);
bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo* pInfo);

2
include/os/osDef.h
View File

@ -220,7 +220,7 @@ void syslog(int unused, const char *format, ...);
// Linux, length of name must <= 16 (the last '\0' included)
#define setThreadName(name) \
do { \
prctl(PR_SET_NAME, (name)); \
(void)prctl(PR_SET_NAME, (name)); \
} while (0)
#define getThreadName(name) \
do { \

1
include/os/osMemory.h
View File

@ -55,6 +55,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size);
#define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n))
#define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n))
#define TAOS_MEMMOVE(_d, _s, _n) ((void)memmove(_d, _s, _n))
#define taosMemoryFreeClear(ptr) \
do { \

1
include/os/osString.h
View File

@ -60,6 +60,7 @@ typedef enum { M2C = 0, C2M } ConvType;
} while (0)
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);

5
include/util/taoserror.h
View File

@ -53,7 +53,7 @@ int32_t taosGetErrSize();
#define terrln (*taosGetErrln())
#define SET_ERROR_MSG(MSG, ...) \
snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
(void)snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
#define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
@ -224,6 +224,7 @@ int32_t taosGetErrSize();
// #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x
#define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315)
#define TSDB_CODE_MND_INVALID_PLATFORM TAOS_DEF_ERROR_CODE(0, 0x0316)
#define TSDB_CODE_MND_RETURN_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x0317)
// mnode-sdb
#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal
@ -555,6 +556,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D)
#define TSDB_CODE_TDB_INCONSISTENT_DB_ID TAOS_DEF_ERROR_CODE(0, 0x061E)
// query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)
@ -596,6 +598,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
#define TSDB_CODE_QRY_FILTER_NOT_SUPPORT_TYPE TAOS_DEF_ERROR_CODE(0, 0x0734)
// grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)

3
include/util/thash.h
View File

@ -28,8 +28,7 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *);
#define HASH_KEY_ALREADY_EXISTS (-2)
#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
#define HASH_NODE_EXIST(code) (code == TSDB_CODE_DUP_KEY)
/**
* murmur hash algorithm

2
include/util/tlist.h
View File

@ -224,7 +224,7 @@ void tdListInit(SList *list, int32_t eleSize);
void tdListEmpty(SList *list);
SList *tdListNew(int32_t eleSize);
void *tdListFree(SList *list);
void *tdListFreeP(SList *list, FDelete fp);
void tdListFreeP(SList *list, FDelete fp);
void tdListPrependNode(SList *list, SListNode *node);
void tdListAppendNode(SList *list, SListNode *node);
int32_t tdListPrepend(SList *list, void *data);

2
include/util/tscalablebf.h
View File

@ -34,7 +34,7 @@ typedef struct SScalableBf {
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes);
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);

1
packaging/deb/DEBIAN/postinst
View File

@ -8,6 +8,7 @@ if command -v sudo > /dev/null; then
csudo="sudo "
fi
${csudo}mkdir -p ${insmetaPath}
${csudo}chmod -R 744 ${insmetaPath}
cd ${insmetaPath}
${csudo}./post.sh

1
packaging/deb/DEBIAN/prerm
View File

@ -33,6 +33,7 @@ else
${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :

13
packaging/deb/makedeb.sh
View File

@ -76,11 +76,11 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || :
fi
if [ -f "%{_compiledir}/../../../explorer/target/taos-explorer.service" ]; then
cp %{_compiledir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || :
if [ -f "${compile_dir}/../../../explorer/target/taos-explorer.service" ]; then
cp ${compile_dir}/../../../explorer/target/taos-explorer.service ${pkg_dir}${install_home_path}/cfg || :
fi
if [ -f "%{_compiledir}/../../../explorer/server/example/explorer.toml" ]; then
cp %{_compiledir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || :
if [ -f "${compile_dir}/../../../explorer/server/example/explorer.toml" ]; then
cp ${compile_dir}/../../../explorer/server/example/explorer.toml ${pkg_dir}${install_home_path}/cfg || :
fi
cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin
@ -90,6 +90,11 @@ cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_pat
cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin
cp ${top_dir}/../enterprise/packaging/start-all.sh ${pkg_dir}${install_home_path}/bin
cp ${top_dir}/../enterprise/packaging/stop-all.sh ${pkg_dir}${install_home_path}/bin
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/start-all.sh
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" ${pkg_dir}${install_home_path}/bin/stop-all.sh
cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin
cp ${compile_dir}/build/bin/udfd ${pkg_dir}${install_home_path}/bin

4
packaging/rpm/makerpm.sh
View File

@ -70,8 +70,8 @@ else
fi
cd ${top_dir}
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper -t ver-${tdengine_ver}"
taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper -t ver-${tdengine_ver}`
echo "taoskeeper_binary: ${taoskeeper_binary}"
cd ${package_dir}

8
packaging/rpm/tdengine.spec
View File

@ -56,6 +56,10 @@ mkdir -p %{buildroot}%{homepath}/include
#mkdir -p %{buildroot}%{homepath}/init.d
mkdir -p %{buildroot}%{homepath}/script
if [ -f %{_compiledir}/../packaging/cfg/taosd.service ]; then
cp %{_compiledir}/../packaging/cfg/taosd.service %{buildroot}%{homepath}/cfg
fi
cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg
if [ -f %{_compiledir}/test/cfg/taosadapter.toml ]; then
cp %{_compiledir}/test/cfg/taosadapter.toml %{buildroot}%{homepath}/cfg
@ -91,6 +95,10 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin
cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin
cp %{_compiledir}/../../enterprise/packaging/start-all.sh %{buildroot}%{homepath}/bin
cp %{_compiledir}/../../enterprise/packaging/stop-all.sh %{buildroot}%{homepath}/bin
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/start-all.sh
sed -i "s/versionType=\"enterprise\"/versionType=\"community\"/g" %{buildroot}%{homepath}/bin/stop-all.sh
if [ -f %{_compiledir}/../../../explorer/target/release/taos-explorer ]; then
cp %{_compiledir}/../../../explorer/target/release/taos-explorer %{buildroot}%{homepath}/bin

63
packaging/tools/install.sh
View File

@ -156,7 +156,7 @@ done
#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh)
tools=(${clientName} ${benchmarkName} ${dumpName} ${demoName} remove.sh udfd set_core.sh TDinsight.sh start_pre.sh start-all.sh stop-all.sh)
if [ "${verMode}" == "cluster" ]; then
services=(${serverName} ${adapterName} ${xname} ${explorerName} ${keeperName})
elif [ "${verMode}" == "edge" ]; then
@ -221,6 +221,8 @@ function install_bin() {
${csudo}cp -r ${script_dir}/bin/remove.sh ${install_main_dir}/bin
else
${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin
${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/bin
${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/bin
fi
if [[ "${verMode}" == "cluster" && "${verType}" != "client" ]]; then
@ -576,7 +578,9 @@ function install_taosd_config() {
${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$serverFqdn/" ${script_dir}/cfg/${configFile}
${csudo}echo "monitor 1" >>${script_dir}/cfg/${configFile}
${csudo}echo "monitorFQDN ${serverFqdn}" >>${script_dir}/cfg/${configFile}
${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile}
if [ "$verMode" == "cluster" ]; then
${csudo}echo "audit 1" >>${script_dir}/cfg/${configFile}
fi
if [ -f "${configDir}/${configFile}" ]; then
${csudo}cp ${fileName} ${configDir}/${configFile}.new
@ -594,6 +598,7 @@ function install_config() {
[ ! -z $1 ] && return 0 || : # only install client
if ((${update_flag} == 1)); then
install_taosd_config
return 0
fi
@ -651,9 +656,7 @@ function install_data() {
function install_connector() {
if [ -d "${script_dir}/connector/" ]; then
${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector"
${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/ || echo "failed to copy start-all.sh"
${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/ || echo "failed to copy stop-all.sh"
${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector"
${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md"
fi
}
@ -910,39 +913,36 @@ function updateProduct() {
echo
echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}"
[ -f ${configDir}/${adapterName}.toml ] && [ -f ${installDir}/bin/${adapterName} ] &&
echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml"
if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml"
fi
echo -e "${GREEN_DARK}To configure ${adapterName} ${NC}\t: edit ${configDir}/${adapterName}.toml"
echo -e "${GREEN_DARK}To configure ${explorerName} ${NC}\t: edit ${configDir}/explorer.toml"
if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}"
echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}"
elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}"
echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}"
else
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ./${serverName}${NC}"
echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ./${serverName}${NC}"
[ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}"
fi
echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
fi
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo
echo "${productName} is updated successfully!"
echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo start all the components : ./start-all.sh${NC}"
fi
echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}"
echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}"
fi
else
@ -1007,39 +1007,36 @@ function installProduct() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}\t\t: edit ${configDir}/${configFile}"
[ -f ${configDir}/${clientName}adapter.toml ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To configure ${clientName}Adapter ${NC}\t: edit ${configDir}/${clientName}adapter.toml"
if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml"
fi
echo -e "${GREEN_DARK}To configure ${clientName}-explorer ${NC}\t: edit ${configDir}/explorer.toml"
if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}systemctl start ${serverName}${NC}"
echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName}adapter ${NC}"
elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${csudo}service ${serverName} start${NC}"
echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/${clientName}adapter.service ] && [ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${csudo}service ${clientName}adapter start${NC}"
else
echo -e "${GREEN_DARK}To start ${productName} ${NC}\t\t: ${serverName}${NC}"
echo -e "${GREEN_DARK}To start ${productName} server ${NC}\t: ${serverName}${NC}"
[ -f ${installDir}/bin/${clientName}adapter ] &&
echo -e "${GREEN_DARK}To start ${clientName}Adapter ${NC}\t\t: ${clientName}adapter ${NC}"
fi
echo -e "${GREEN_DARK}To enable ${clientName}keeper ${NC}\t\t: sudo systemctl enable ${clientName}keeper ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}keeper ${NC}\t\t: sudo systemctl start ${clientName}keeper ${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "${GREEN_DARK}To start ${clientName}x ${NC}\t\t\t: sudo systemctl start ${clientName}x ${NC}"
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
fi
echo -e "${GREEN_DARK}To start ${clientName}-explorer ${NC}\t\t: sudo systemctl start ${clientName}-explorer ${NC}"
echo
echo "${productName} is installed successfully!"
echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : ${clientName} -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo start all the components : sudo ./start-all.sh${NC}"
fi
echo -e "\033[44;32;1mTo access ${productName} : ${clientName} -h $serverFqdn${NC}"
if [ "$verMode" == "cluster" ]; then
echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}"
echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}"
fi
echo

2
packaging/tools/makepkg.sh
View File

@ -232,7 +232,7 @@ fi
if [ "$verMode" == "cluster" ]; then
sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" remove_temp.sh
sed -i "s/PREFIX=\"taos\"/PREFIX=\"${clientName2}\"/g" remove_temp.sh
sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" remove_temp.sh
mv remove_temp.sh ${install_dir}/bin/remove.sh
fi

127
packaging/tools/post.sh
View File

@ -104,6 +104,8 @@ fi
function log_print(){
now=$(date +"%D %T")
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
${csudo} touch ${install_log_path}
echo "$now $1" >> ${install_log_path}
}
@ -235,11 +237,14 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taos-explorer || :
${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/rmtaos || :
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/*explorer || :
${csudo}rm -f ${bin_link_dir}/start-all.sh || :
${csudo}rm -f ${bin_link_dir}/stop-all.sh || :
${csudo}chmod 0555 ${bin_dir}/*
@ -275,8 +280,14 @@ function install_bin() {
if [ -x ${bin_dir}/taoskeeper ]; then
${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/*explorer ]; then
${csudo}ln -s ${bin_dir}/*explorer ${bin_link_dir}/*explorer 2>>${install_log_path} || return 1
if [ -x ${bin_dir}/taos-explorer ]; then
${csudo}ln -s ${bin_dir}/taos-explorer ${bin_link_dir}/taos-explorer 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/start-all.sh ]; then
${csudo}ln -s ${bin_dir}/start-all.sh ${bin_link_dir}/start-all.sh 2>>${install_log_path} || return 1
fi
if [ -x ${bin_dir}/stop-all.sh ]; then
${csudo}ln -s ${bin_dir}/stop-all.sh ${bin_link_dir}/stop-all.sh 2>>${install_log_path} || return 1
fi
log_print "install bin success"
}
@ -474,6 +485,22 @@ function install_taoskeeper_config() {
${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir}
}
function install_taos-explorer_config() {
if [ ! -f "${cfg_install_dir}/explorer.toml" ]; then
[ ! -d ${cfg_install_dir} ] &&
${csudo}${csudo}mkdir -p ${cfg_install_dir}
[ -f ${cfg_dir}/explorer.toml ] && ${csudo}cp ${cfg_dir}/explorer.toml ${cfg_install_dir}
[ -f ${cfg_install_dir}/explorer.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/explorer.toml
fi
[ -f ${cfg_dir}/explorer.toml ] &&
${csudo}mv ${cfg_dir}/explorer.toml ${cfg_dir}/explorer.toml.new
[ -f ${cfg_install_dir}/explorer.toml ] &&
${csudo}ln -s ${cfg_install_dir}/explorer.toml ${cfg_dir}
}
function install_config() {
log_print "start install config from ${cfg_dir} to ${cfg_install_dir}"
if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then
@ -609,30 +636,12 @@ function clean_service_on_systemd() {
function install_service_on_systemd() {
clean_service_on_systemd
[ -f ${script_dir}/../cfg/taosd.service ] &&\
${csudo}cp ${script_dir}/../cfg/taosd.service \
${service_config_dir}/ || :
taosd_service_config="${service_config_dir}/taosd.service"
${csudo}bash -c "echo '[Unit]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}"
${csudo}bash -c "echo 'After=network-online.target' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Wants=network-online.target' >> ${taosd_service_config}"
${csudo}bash -c "echo >> ${taosd_service_config}"
${csudo}bash -c "echo '[Service]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Type=simple' >> ${taosd_service_config}"
${csudo}bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}"
${csudo}bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}"
${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}"
${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StandardOutput=null' >> ${taosd_service_config}"
${csudo}bash -c "echo 'Restart=always' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}"
${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}"
${csudo}bash -c "echo >> ${taosd_service_config}"
${csudo}bash -c "echo '[Install]' >> ${taosd_service_config}"
${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}"
${csudo}systemctl daemon-reload
${csudo}systemctl enable taosd
}
@ -652,6 +661,11 @@ function install_service_on_launchctl() {
${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || :
fi
if [ -f ${install_main_dir}/service/com.taosdata.taos-explorer.plist ]; then
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist > /dev/null 2>&1 || :
${csudo}cp ${install_main_dir}/service/com.taosdata.taos-explorer.plist /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || :
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taos-explorer.plist || :
fi
}
function install_taosadapter_service() {
@ -672,6 +686,15 @@ function install_taoskeeper_service() {
fi
}
function install_taos-explorer_service() {
if ((${service_mod}==0)); then
[ -f ${script_dir}/../cfg/taos-explorer.service ] &&\
${csudo}cp ${script_dir}/../cfg/taos-explorer.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
}
function install_service() {
log_print "start install service"
if [ "$osType" != "Darwin" ]; then
@ -748,8 +771,10 @@ function install_TDengine() {
install_config
install_taosadapter_config
install_taoskeeper_config
install_taos-explorer_config
install_taosadapter_service
install_taoskeeper_service
install_taos-explorer_service
install_service
install_app
@ -757,40 +782,38 @@ function install_TDengine() {
#echo
#echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo
echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg"
echo -e "${GREEN_DARK}To configure TDengine ${NC}\t: edit /etc/taos/taos.cfg"
echo -e "${GREEN_DARK}To configure taosAdapter ${NC}\t: edit /etc/taos/taosadapter.toml"
echo -e "${GREEN_DARK}To configure taos-explorer ${NC}\t: edit /etc/taos/explorer.toml"
if ((${service_mod}==0)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}systemctl start taosd${NC}"
echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}systemctl start taosd${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}systemctl start taosadapter${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}systemctl start taoskeeper${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}systemctl start taos-explorer${NC}"
elif ((${service_mod}==1)); then
echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}"
echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ${csudo}update-rc.d taosd default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd start ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ${csudo}update-rc.d taosadapter default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taosadapter ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ${csudo}update-rc.d taoskeeper default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taoskeeper ${RED} after${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ${csudo}update-rc.d taos-explorer default ${RED} for the first time${NC}"
echo -e " : ${csudo}service taosd taos-explorer ${RED} after${NC}"
else
echo -e "${GREEN_DARK}To start TDengine ${NC}: ./taosd${NC}"
fi
if [ ! -z "$firstEp" ]; then
tmpFqdn=${firstEp%%:*}
substr=":"
if [[ $firstEp =~ $substr ]];then
tmpPort=${firstEp#*:}
else
tmpPort=""
fi
if [[ "$tmpPort" != "" ]];then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
else
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
fi
echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
echo
elif [ ! -z "$serverFqdn" ]; then
echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}"
echo
echo -e "${GREEN_DARK}To start TDengine server ${NC}\t: ./taosd${NC}"
echo -e "${GREEN_DARK}To start taosAdapter ${NC}\t: ./taosadapter${NC}"
echo -e "${GREEN_DARK}To start taoskeeper ${NC}\t: ./taoskeeper${NC}"
echo -e "${GREEN_DARK}To start taos-explorer ${NC}\t: ./taos-explorer${NC}"
fi
log_print "install TDengine successfully!"
echo
echo -e "\033[44;32;1mTDengine is installed successfully!${NC}"
echo "TDengine is installed successfully!"
echo
echo -e "\033[44;32;1mTo start all the components : sudo start-all.sh${NC}"
echo -e "\033[44;32;1mTo access ${productName} Commnd Line Interface : taos -h $serverFqdn${NC}"
echo -e "\033[44;32;1mTo access ${productName} Graphic User Interface : http://$serverFqdn:6060${NC}"
}

359
packaging/tools/remove.sh
View File

@ -31,25 +31,22 @@ else
lib64_link_dir="/usr/local/lib"
inc_link_dir="/usr/local/include"
fi
serverName="taosd"
clientName="taos"
uninstallScript="rmtaos"
PREFIX="taos"
serverName="${PREFIX}d"
clientName="${PREFIX}"
uninstallScript="rm${PREFIX}"
adapterName="${PREFIX}adapter"
demoName="${PREFIX}demo"
benchmarkName="${PREFIX}Benchmark"
dumpName="${PREFIX}dump"
keeperName="${PREFIX}keeper"
xName="${PREFIX}x"
explorerName="${PREFIX}-explorer"
tarbitratorName="tarbitratord"
productName="TDengine"
serverName2="taosd"
clientName2="taos"
productName2="TDengine"
adapterName2="${clientName2}adapter"
demoName2="${clientName2}demo"
benchmarkName2="${clientName2}Benchmark"
dumpName2="${clientName2}dump"
keeperName2="${clientName2}keeper"
xName2="${clientName2}x"
explorerName2="${clientName2}-explorer"
uninstallScript2="rm${clientName2}"
installDir="/usr/local/${clientName2}"
installDir="/usr/local/${PREFIX}"
#install main path
install_main_dir=${installDir}
@ -57,15 +54,11 @@ data_link_dir=${installDir}/data
log_link_dir=${installDir}/log
cfg_link_dir=${installDir}/cfg
local_bin_link_dir="/usr/local/bin"
service_config_dir="/etc/systemd/system"
taos_service_name=${serverName2}
taosadapter_service_name="${clientName2}adapter"
tarbitrator_service_name="tarbitratord"
config_dir="/etc/${clientName2}"
config_dir="/etc/${PREFIX}"
services=(${PREFIX}"d" ${PREFIX}"adapter" ${PREFIX}"x" ${PREFIX}"-explorer" ${PREFIX}"keeper")
tools=(${PREFIX} ${PREFIX}"Benchmark" ${PREFIX}"dump" ${PREFIX}"demo" udfd set_core.sh TDinsight.sh $uninstallScript start-all.sh stop-all.sh)
csudo=""
if command -v sudo >/dev/null; then
@ -92,215 +85,26 @@ else
service_mod=2
fi
function kill_taosadapter() {
pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_taosd() {
pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_tarbitrator() {
pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function clean_bin() {
# Remove link
${csudo}rm -f ${bin_link_dir}/${clientName} || :
${csudo}rm -f ${bin_link_dir}/${serverName} || :
echo "${serverName} is removed successfully"
${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/${adapterName2} || :
echo "${adapterName2} is removed successfully"
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
${csudo}rm -f ${bin_link_dir}/tarbitrator || :
${csudo}rm -f ${bin_link_dir}/set_core || :
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
${csudo}rm -f ${bin_link_dir}/${keeperName2} || :
if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
${csudo}rm -f ${bin_link_dir}/${xName2} || :
${csudo}rm -f ${bin_link_dir}/${explorerName2} || :
${csudo}rm -f ${bin_link_dir}/${clientName2} || :
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || :
fi
}
function clean_local_bin() {
${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || :
${csudo}rm -f ${local_bin_link_dir}/${demoName2} || :
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
# Remove link
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {
# Remove link
${csudo}rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo}rm -rf ${log_link_dir} || :
}
function clean_service_on_systemd() {
taosd_service_config="${service_config_dir}/${taos_service_name}.service"
if systemctl is-active --quiet ${taos_service_name}; then
echo "${taos_service_name} is running, stopping it..."
${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${taosd_service_config}
taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service"
if systemctl is-active --quiet ${taosadapter_service_name}; then
echo "${clientName2}Adapter is running, stopping it..."
${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null
[ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service"
if systemctl is-active --quiet ${tarbitrator_service_name}; then
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null
}
function clean_service_on_sysvinit() {
if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then
echo "${serverName2} is running, stopping it..."
${csudo}service ${serverName} stop || :
fi
if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then
echo "${productName2} tarbitrator is running, stopping it..."
${csudo}service tarbitratord stop || :
fi
if ((${initd_mod} == 1)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}chkconfig --del ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}chkconfig --del tarbitratord || :
fi
elif ((${initd_mod} == 2)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}insserv -r ${serverName} || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}insserv -r tarbitratord || :
fi
elif ((${initd_mod} == 3)); then
if [ -e ${service_config_dir}/${serverName} ]; then
${csudo}update-rc.d -f ${serverName} remove || :
fi
if [ -e ${service_config_dir}/tarbitratord ]; then
${csudo}update-rc.d -f tarbitratord remove || :
fi
fi
${csudo}rm -f ${service_config_dir}/${serverName} || :
${csudo}rm -f ${service_config_dir}/tarbitratord || :
if $(which init &>/dev/null); then
${csudo}init q || :
fi
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
}
function clean_service() {
if ((${service_mod} == 0)); then
clean_service_on_systemd
elif ((${service_mod} == 1)); then
clean_service_on_sysvinit
else
if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
fi
kill_taosadapter
kill_taosd
kill_tarbitrator
fi
}
function remove_data_and_config() {
data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$data_dir" == X"" ]; then
data_dir="/var/lib/taos"
fi
log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$log_dir" == X"" ]; then
log_dir="/var/log/taos"
fi
[ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}/*
[ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}/*
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}/*
}
_kill_service_of() {
kill_service_of() {
_service=$1
pid=$(ps -ef | grep "$_service" | grep -v "grep" | awk '{print $2}')
pid=$(ps -ef | grep $_service | grep -v grep | grep -v $uninstallScript | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
_clean_service_on_systemd_of() {
clean_service_on_systemd_of() {
_service=$1
_service_config="${service_config_dir}/${_service}.service"
if systemctl is-active --quiet ${_service}; then
echo "taoskeeper is running, stopping it..."
echo "${_service} is running, stopping it..."
${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null
fi
${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null
${csudo}rm -f ${_service_config}
}
_clean_service_on_sysvinit_of() {
clean_service_on_sysvinit_of() {
_service=$1
if pidof ${_service} &>/dev/null; then
echo "${_service} is running, stopping it..."
@ -327,45 +131,99 @@ _clean_service_on_sysvinit_of() {
fi
}
_clean_service_of() {
clean_service_of() {
_service=$1
if ((${service_mod} == 0)); then
_clean_service_on_systemd_of $_service
clean_service_on_systemd_of $_service
elif ((${service_mod} == 1)); then
_clean_service_on_sysvinit_of $_service
clean_service_on_sysvinit_of $_service
else
_kill_service_of $_service
kill_service_of $_service
fi
}
remove_taoskeeper() {
# remove taoskeeper bin
_clean_service_of taoskeeper
[ -e "${bin_link_dir}/taoskeeper" ] && ${csudo}rm -rf ${bin_link_dir}/taoskeeper
[ -e "${installDir}/taoskeeper" ] && ${csudo}rm -rf ${installDir}/taoskeeper
[ -e "${cfg_link_dir}/metrics.toml" ] || ${csudo}rm -rf ${cfg_link_dir}/metrics.toml
echo "taosKeeper is removed successfully!"
}
function uninstall_taosx() {
if [ -f ${installDir}/uninstall.sh ]; then
cd ${installDir}
bash uninstall.sh
remove_service_of() {
_service=$1
clean_service_of ${_service}
if [[ -e "${bin_link_dir}/${_service}" || -e "${installDir}/bin/${_service}" || -e "${local_bin_link_dir}/${_service}" ]]; then
${csudo}rm -rf ${bin_link_dir}/${_service}
${csudo}rm -rf ${installDir}/bin/${_service}
${csudo}rm -rf ${local_bin_link_dir}/${_service}
echo "${_service} is removed successfully!"
fi
}
if [ "$verMode" == "cluster" ]; then
uninstall_taosx
fi
remove_tools_of() {
_tool=$1
kill_service_of ${_tool}
[ -e "${bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${bin_link_dir}/${_tool} || :
[ -e "${installDir}/bin/${_tool}" ] && ${csudo}rm -rf ${installDir}/bin/${_tool} || :
[ -e "${local_bin_link_dir}/${_tool}" ] && ${csudo}rm -rf ${local_bin_link_dir}/${_tool} || :
}
remove_taoskeeper
# Stop service and disable booting start.
clean_service
# Remove binary file and links
clean_bin
# Remove links of local bin
clean_local_bin
# Remove header file.
remove_bin() {
for _service in "${services[@]}"; do
remove_service_of ${_service}
done
for _tool in "${tools[@]}"; do
remove_tools_of ${_tool}
done
}
function clean_lib() {
# Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
}
function clean_header() {
# Remove link
${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || :
}
function clean_config() {
# Remove link
${csudo}rm -f ${cfg_link_dir}/* || :
}
function clean_log() {
# Remove link
${csudo}rm -rf ${log_link_dir} || :
}
function clean_service_on_launchctl() {
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || :
}
function remove_data_and_config() {
data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$data_dir" == X"" ]; then
data_dir="/var/lib/${PREFIX}"
fi
log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}`
if [ X"$log_dir" == X"" ]; then
log_dir="/var/log/${PREFIX}"
fi
[ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}
[ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}
[ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}
}
remove_bin
clean_header
# Remove lib file
clean_lib
@ -375,8 +233,7 @@ clean_log
clean_config
# Remove data link directory
${csudo}rm -rf ${data_link_dir} || :
${csudo}rm -rf ${install_main_dir}
${csudo}rm -rf ${install_main_dir} || :
if [[ -e /etc/os-release ]]; then
osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
else
@ -394,6 +251,7 @@ elif echo $osinfo | grep -qwi "centos"; then
${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || :
fi
if [ "$osType" = "Darwin" ]; then
clean_service_on_launchctl
${csudo}rm -rf /Applications/TDengine.app
fi
@ -411,6 +269,7 @@ if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then
fi
fi
command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true
echo
echo "${productName2} is removed successfully!"
echo "${productName} is removed successfully!"
echo

7
packaging/tools/remove_client.sh
View File

@ -38,9 +38,10 @@ if command -v sudo > /dev/null; then
fi
function kill_client() {
if [ -n "$(ps aux | grep -v grep | grep ${clientName})" ]; then
${csudo}kill -9 $pid || :
fi
pid=$(ps -ef | grep ${clientName2} | grep -v grep | grep -v $uninstallScript2 | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function clean_bin() {

6
source/client/src/clientEnv.c
View File

@ -885,7 +885,11 @@ void taos_init_imp(void) {
taosSetCoreDump(true);
#endif
initTaskQueue();
if (initTaskQueue() != 0){
tscInitRes = -1;
tscError("failed to init task queue");
return;
}
fmFuncMgtInit();
nodesInitAllocatorSet();

2
source/client/src/clientHb.c
View File

@ -744,7 +744,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
}
if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) {
if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) != 0) {
taosMemoryFree(user);
code = terrno ? terrno : TSDB_CODE_APP_ERROR;
goto _return;

3
source/client/src/clientImpl.c
View File

@ -2007,7 +2007,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (tTagIsJson(data)) {
char* jsonString = parseTagDatatoJson(data);
char* jsonString = NULL;
parseTagDatatoJson(data, &jsonString);
STR_TO_VARSTR(dst, jsonString);
taosMemoryFree(jsonString);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"

2
source/client/src/clientMonitor.c
View File

@ -691,7 +691,7 @@ static void* monitorThreadFunc(void* param) {
tscDebug("monitorThreadFunc start");
int64_t quitTime = 0;
while (1) {
if (atomic_load_32(&slowLogFlag) > 0 > 0) {
if (atomic_load_32(&slowLogFlag) > 0) {
if (quitCnt == 0) {
monitorSendAllSlowLogAtQuit();
if (quitCnt == 0) {

1222
source/client/src/clientRawBlockWrite.c
View File

File diff suppressed because it is too large Load Diff

20
source/client/test/CMakeLists.txt
View File

@ -23,11 +23,11 @@ TARGET_LINK_LIBRARIES(
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
)
ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
TARGET_LINK_LIBRARIES(
clientMonitorTest
PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
)
#ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
#TARGET_LINK_LIBRARIES(
# clientMonitorTest
# PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
#)
TARGET_INCLUDE_DIRECTORIES(
clientTest
@ -47,11 +47,11 @@ TARGET_INCLUDE_DIRECTORIES(
PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
)
TARGET_INCLUDE_DIRECTORIES(
clientMonitorTest
PUBLIC "${TD_SOURCE_DIR}/include/client/"
PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
)
#TARGET_INCLUDE_DIRECTORIES(
# clientMonitorTest
# PUBLIC "${TD_SOURCE_DIR}/include/client/"
# PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
#)
add_test(
NAME smlTest

26
source/common/src/rsync.c
View File

@ -54,10 +54,11 @@ static void changeDirFromWindowsToLinux(char* from, char* to) {
#endif
static int32_t generateConfigFile(char* confDir) {
int32_t code = 0;
TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
#ifdef WINDOWS
@ -92,7 +93,8 @@ static int32_t generateConfigFile(char* confDir) {
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
taosCloseFile(&pFile);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
taosCloseFile(&pFile);
@ -129,11 +131,13 @@ void stopRsync() {
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
}
void startRsync() {
int32_t startRsync() {
int32_t code = 0;
if (taosMulMkDir(tsCheckpointBackupDir) != 0) {
uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir,
ERRNO_ERR_DATA);
return;
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
removeEmptyDir();
@ -141,9 +145,9 @@ void startRsync() {
char confDir[PATH_MAX] = {0};
snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
int32_t code = generateConfigFile(confDir);
code = generateConfigFile(confDir);
if (code != 0) {
return;
return code;
}
char cmd[PATH_MAX] = {0};
@ -152,9 +156,11 @@ void startRsync() {
code = system(cmd);
if (code != 0) {
uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else {
uInfo("[rsync] cmd:%s start server successful", cmd);
}
return code;
}
int32_t uploadByRsync(const char* id, const char* path) {
@ -199,6 +205,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) {
uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else {
int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -242,6 +249,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) {
uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else {
int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -283,13 +291,13 @@ int32_t downloadRsync(const char* id, const char* path) {
uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
path, times, code, ERRNO_ERR_DATA);
taosSsleep(1);
code = TAOS_SYSTEM_ERROR(errno);
} else {
int32_t el = taosGetTimestampMs() - st;
uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
break;
}
}
return code;
}
@ -298,7 +306,7 @@ int32_t deleteRsync(const char* id) {
int32_t code = taosMkDir(tmp);
if (code != 0) {
uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
char command[PATH_MAX] = {0};
@ -310,7 +318,7 @@ int32_t deleteRsync(const char* id) {
taosRemoveDir(tmp);
if (code != 0) {
uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1;
return TAOS_SYSTEM_ERROR(errno);
}
uDebug("[rsync] delete data:%s successful", id);

12
source/common/src/tdatablock.c
View File

@ -1523,9 +1523,9 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
}
void* blockDataDestroy(SSDataBlock* pBlock) {
void blockDataDestroy(SSDataBlock* pBlock) {
if (pBlock == NULL) {
return NULL;
return;
}
if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) {
@ -1535,7 +1535,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
blockDataFreeRes(pBlock);
taosMemoryFreeClear(pBlock);
return NULL;
}
// todo remove it
@ -2468,19 +2467,18 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, char* cname) {
if (stbFullName[0] == 0) {
terrno = TSDB_CODE_INVALID_PARA;
return TSDB_CODE_FAILED;
return TSDB_CODE_INVALID_PARA;
}
SArray* tags = taosArrayInit(0, sizeof(SSmlKv));
if (tags == NULL) {
return TSDB_CODE_FAILED;
return TSDB_CODE_OUT_OF_MEMORY;
}
if (cname == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
taosArrayDestroy(tags);
return TSDB_CODE_FAILED;
return terrno;
}
int8_t type = TSDB_DATA_TYPE_UBIGINT;

58
source/common/src/tglobal.c
View File

@ -1370,58 +1370,68 @@ static int32_t taosCheckGlobalCfg() {
return 0;
}
static int32_t cfgInitWrapper(SConfig **pCfg) {
if (*pCfg == NULL) {
*pCfg = cfgInit();
if (*pCfg == NULL) {
return terrno;
}
}
return 0;
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) {
if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
int32_t code = cfgInitWrapper(&tsCfg);
if (tsc) {
if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
} else {
if (taosAddClientCfg(tsCfg) != 0) return -1;
if (taosAddServerCfg(tsCfg) != 0) return -1;
if (taosAddClientLogCfg(tsCfg) != 0) return -1;
if (taosAddServerLogCfg(tsCfg) != 0) return -1;
if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if ((code = taosAddServerCfg(tsCfg)) != 0) return code;
if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
if ((code = taosAddServerLogCfg(tsCfg)) != 0) return code;
}
taosAddSystemCfg(tsCfg);
code = taosAddSystemCfg(tsCfg);
if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr());
if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
uError("failed to load cfg since %s", tstrerror(code));
cfgCleanup(tsCfg);
tsCfg = NULL;
return -1;
return code;
}
if (cfgLoadFromArray(tsCfg, pArgs) != 0) {
uError("failed to load cfg from array since %s", terrstr());
if ((code = cfgLoadFromArray(tsCfg, pArgs)) != 0) {
uError("failed to load cfg from array since %s", tstrerror(code));
cfgCleanup(tsCfg);
tsCfg = NULL;
return -1;
return code;
}
if (tsc) {
if (taosSetClientCfg(tsCfg)) return -1;
if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
} else {
if (taosSetClientCfg(tsCfg)) return -1;
if (taosUpdateServerCfg(tsCfg)) return -1;
if (taosSetServerCfg(tsCfg)) return -1;
if (taosSetReleaseCfg(tsCfg)) return -1;
if (taosSetTfsCfg(tsCfg) != 0) return -1;
if (taosSetS3Cfg(tsCfg) != 0) return -1;
if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
if ((code = taosUpdateServerCfg(tsCfg)) != 0) return code;
if ((code = taosSetServerCfg(tsCfg)) != 0) return code;
if ((code = taosSetReleaseCfg(tsCfg)) != 0) return code;
if ((code = taosSetTfsCfg(tsCfg)) != 0) return code;
if ((code = taosSetS3Cfg(tsCfg)) != 0) return code;
}
taosSetSystemCfg(tsCfg);
if (taosSetFileHandlesLimit() != 0) return -1;
if ((code = taosSetFileHandlesLimit()) != 0) return code;
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) {
return -1;
if ((code = taosCheckGlobalCfg()) != 0) {
return code;
}
return 0;

12
source/common/src/tmsg.c
View File

@ -9258,6 +9258,12 @@ void tOffsetDestroy(void *param) {
taosMemoryFreeClear(pVal->primaryKey.pData);
}
}
void tDeleteSTqOffset(void *param) {
STqOffset *pVal = (STqOffset *)param;
tOffsetDestroy(&pVal->val);
}
int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) {
if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1;
if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1;
@ -9297,12 +9303,12 @@ int32_t tEncodeSTqCheckInfo(SEncoder *pEncoder, const STqCheckInfo *pInfo) {
int32_t tDecodeSTqCheckInfo(SDecoder *pDecoder, STqCheckInfo *pInfo) {
if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1;
if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1;
int32_t sz;
int32_t sz = 0;
if (tDecodeI32(pDecoder, &sz) < 0) return -1;
pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t));
if (pInfo->colIdList == NULL) return -1;
for (int32_t i = 0; i < sz; i++) {
int16_t colId;
int16_t colId = 0;
if (tDecodeI16(pDecoder, &colId) < 0) return -1;
taosArrayPush(pInfo->colIdList, &colId);
}
@ -10822,7 +10828,9 @@ int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES);
if (!pRsp->batchMetaReq) return -1;
pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t));
if (!pRsp->batchMetaLen) return -1;
for (int32_t i = 0; i < size; i++) {
void *pCreate = NULL;
uint64_t len = 0;

77
source/dnode/mgmt/exe/dmMain.c
View File

@ -167,23 +167,23 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-a") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow");
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(global.apolloUrl, argv[i], PATH_MAX);
} else {
printf("'-a' requires a parameter\n");
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true;
@ -191,31 +191,31 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow");
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(global.envFile, argv[i], PATH_MAX);
} else {
printf("'-E' requires a parameter\n");
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true;
if(i < argc - 1) {
if (i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1;
return TSDB_CODE_INVALID_CFG;
}
if (len > ENCRYPT_KEY_LEN) {
printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1;
return TSDB_CODE_INVALID_CFG;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else {
printf("'-y' requires a parameter\n");
return -1;
return TSDB_CODE_INVALID_CFG;
}
} else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true;
@ -310,6 +310,7 @@ static void taosCleanupArgs() {
}
int main(int argc, char const *argv[]) {
int32_t code = 0;
#ifdef TD_JEMALLOC_ENABLED
bool jeBackgroundThread = true;
mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool));
@ -319,10 +320,10 @@ int main(int argc, char const *argv[]) {
return -1;
}
if (dmParseArgs(argc, argv) != 0) {
//printf("failed to start since parse args error\n");
if ((code = dmParseArgs(argc, argv)) != 0) {
// printf("failed to start since parse args error\n");
taosCleanupArgs();
return -1;
return code;
}
#ifdef WINDOWS
@ -335,6 +336,7 @@ int main(int argc, char const *argv[]) {
return 0;
}
int mainWindows(int argc, char **argv) {
int32_t code = 0;
#endif
if (global.generateGrant) {
@ -357,7 +359,7 @@ int mainWindows(int argc, char **argv) {
#if defined(LINUX)
if (global.memDbg) {
int32_t code = taosMemoryDbgInit();
code = taosMemoryDbgInit();
if (code) {
printf("failed to init memory dbg, error:%s\n", tstrerror(code));
return code;
@ -366,16 +368,18 @@ int mainWindows(int argc, char **argv) {
printf("memory dbg enabled\n");
}
#endif
if(global.generateCode) {
if (global.generateCode) {
bool toLogFile = false;
if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){
encryptError("failed to generate encrypt code since dataDir can not be set from cfg file");
return -1;
if ((code = taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs)) != 0) {
encryptError("failed to generate encrypt code since dataDir can not be set from cfg file,reason:%s",
tstrerror(code));
return code;
};
if(dmCheckRunning(tsDataDir) == NULL) {
encryptError("failed to generate encrypt code since taosd is running, please stop it first");
return -1;
TdFilePtr pFile;
if ((code = dmCheckRunning(tsDataDir, &pFile)) != 0) {
encryptError("failed to generate encrypt code since taosd is running, please stop it first, reason:%s",
tstrerror(code));
return code;
}
int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile);
taosCloseLog();
@ -383,30 +387,30 @@ int mainWindows(int argc, char **argv) {
return ret;
}
if (dmInitLog() != 0) {
if ((code = dmInitLog()) != 0) {
printf("failed to start since init log error\n");
taosCleanupArgs();
return -1;
return code;
}
dmPrintArgs(argc, argv);
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
if ((code = taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0)) != 0) {
dError("failed to start since read config error");
taosCloseLog();
taosCleanupArgs();
return -1;
return code;
}
if (taosConvInit() != 0) {
if ((code = taosConvInit()) != 0) {
dError("failed to init conv");
taosCloseLog();
taosCleanupArgs();
return -1;
return code;
}
if (global.checkS3) {
int32_t code = dmCheckS3();
code = dmCheckS3();
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
@ -435,31 +439,32 @@ int mainWindows(int argc, char **argv) {
osSetProcPath(argc, (char **)argv);
taosCleanupArgs();
if(dmGetEncryptKey() != 0){
if ((code = dmGetEncryptKey()) != 0) {
dError("failed to start since failed to get encrypt key");
taosCloseLog();
taosCleanupArgs();
return -1;
return code;
};
if (dmInit() != 0) {
if (terrno == TSDB_CODE_NOT_FOUND) {
if ((code = dmInit()) != 0) {
if (code == TSDB_CODE_NOT_FOUND) {
dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support");
} else {
dError("failed to init dnode since %s", terrstr());
dError("failed to init dnode since %s", tstrerror(code));
}
taosCleanupCfg();
taosCloseLog();
taosConvDestroy();
return -1;
return code;
}
dInfo("start to init service");
dmSetSignalHandle();
tsDndStart = taosGetTimestampMs();
tsDndStartOsUptime = taosGetOsUptime();
int32_t code = dmRun();
code = dmRun();
dInfo("shutting down the service");
dmCleanup();

127
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -45,7 +45,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer};
int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen);
void *pHead = rpcMallocCont(contLen);
tSerializeRetrieveIpWhite(pHead, contLen, &req);
SRpcMsg rpcMsg = {.pCont = pHead,
@ -116,7 +116,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite;
req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0;
req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
req.clusterCfg.monitorParas.tsEnableMonitor = tsEnableMonitor;
req.clusterCfg.monitorParas.tsMonitorInterval = tsMonitorInterval;
req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope;
@ -146,7 +146,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen);
void *pHead = rpcMallocCont(contLen);
tSerializeSStatusReq(pHead, contLen, &req);
tFreeSStatusReq(&req);
@ -207,18 +207,26 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
SConfig *pCfg = taosGetCfg();
cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
taosCfgDynamicOptions(pCfg, cfgReq.config, true);
return 0;
code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
if (code != 0) {
if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) {
code = 0;
} else {
return code;
}
}
return taosCfgDynamicOptions(pCfg, cfgReq.config, true);
}
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@ -251,7 +259,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
pStatus->details[0] = 0;
SMonMloadInfo minfo = {0};
SMonMloadInfo minfo = {0};
(*pMgmt->getMnodeLoadsFp)(&minfo);
if (minfo.isMnode &&
(minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) {
@ -276,32 +284,49 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
}
int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
dDebug("server run status req is received");
SServerStatusRsp statusRsp = {0};
dmGetServerRunStatus(pMgmt, &statusRsp);
pMsg->info.rsp = NULL;
pMsg->info.rspLen = 0;
SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
// rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
}
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
// rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
}
rspLen = tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
if (rspLen < 0) {
return TSDB_CODE_INVALID_MSG;
}
tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
pMsg->info.rsp = pRsp;
pMsg->info.rspLen = rspLen;
return 0;
}
SSDataBlock *dmBuildVariablesBlock(void) {
SSDataBlock * pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
size_t size = 0;
int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) {
int32_t code = 0;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
size_t size = 0;
const SSysTableMeta *pMeta = NULL;
getInfosDbMeta(&pMeta, &size);
@ -314,52 +339,74 @@ SSDataBlock *dmBuildVariablesBlock(void) {
}
pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData = {0};
colInfoData.info.colId = i + 1;
colInfoData.info.type = pMeta[index].schema[i].type;
colInfoData.info.bytes = pMeta[index].schema[i].bytes;
taosArrayPush(pBlock->pDataBlock, &colInfoData);
if (taosArrayPush(pBlock->pDataBlock, &colInfoData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
pBlock->info.hasVarCol = true;
return pBlock;
_exit:
if (code != 0) {
blockDataDestroy(pBlock);
} else {
*ppBlock = pBlock;
}
return code;
}
int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
/*int32_t code = */dumpConfToDataBlock(pBlock, 1);
int32_t code = dumpConfToDataBlock(pBlock, 1);
if (code != 0) {
return code;
}
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
if (pColInfo == NULL) {
return TSDB_CODE_OUT_OF_RANGE;
}
return TSDB_CODE_SUCCESS;
return colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
}
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t size = 0;
int32_t rowsRead = 0;
int32_t size = 0;
int32_t rowsRead = 0;
int32_t code = 0;
SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
#if 0
if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
code = TSDB_CODE_MND_NO_RIGHTS;
return code;
}
#endif
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
SSDataBlock *pBlock = dmBuildVariablesBlock();
SSDataBlock *pBlock = NULL;
if ((code = dmBuildVariablesBlock(&pBlock)) != 0) {
return code;
}
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
code = dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
if (code != 0) {
blockDataDestroy(pBlock);
return code;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
@ -367,10 +414,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to retrieve data since %s", terrstr());
code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to retrieve data since %s", tstrerror(code));
blockDataDestroy(pBlock);
return -1;
return code;
}
char *pStart = pRsp->data;
@ -404,7 +451,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SArray *dmGetMsgHandles() {
int32_t code = -1;
SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));
if (pArray == NULL) goto _OVER;
if (pArray == NULL) {
return NULL;
}
// Requests handled by DNODE
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
@ -416,7 +465,7 @@ SArray *dmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// Requests handled by MNODE

32
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -18,22 +18,23 @@
#include "libs/function/tudf.h"
static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartStatusThread(pMgmt) != 0) {
return -1;
int32_t code = 0;
if ((code = dmStartStatusThread(pMgmt)) != 0) {
return code;
}
#if defined(TD_ENTERPRISE)
if (dmStartNotifyThread(pMgmt) != 0) {
return -1;
if ((code = dmStartNotifyThread(pMgmt)) != 0) {
return code;
}
#endif
if (dmStartMonitorThread(pMgmt) != 0) {
return -1;
if ((code = dmStartMonitorThread(pMgmt)) != 0) {
return code;
}
if (dmStartAuditThread(pMgmt) != 0) {
return -1;
if ((code = dmStartAuditThread(pMgmt)) != 0) {
return code;
}
if (dmStartCrashReportThread(pMgmt) != 0) {
return -1;
if ((code = dmStartCrashReportThread(pMgmt)) != 0) {
return code;
}
return 0;
}
@ -50,10 +51,10 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) {
}
static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pMgmt->pData = pInput->pData;
@ -70,12 +71,11 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp;
// pMgmt->pData->ipWhiteVer = 0;
if (dmStartWorker(pMgmt) != 0) {
return -1;
if ((code = dmStartWorker(pMgmt)) != 0) {
return code;
}
if (udfStartUdfd(pMgmt->pData->dnodeId) != 0) {
if ((code = udfStartUdfd(pMgmt->pData->dnodeId)) != 0) {
dError("failed to start udfd");
}

34
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -264,12 +264,14 @@ static void *dmCrashReportThreadFp(void *param) {
}
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) {
dError("failed to create status thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create status thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -285,12 +287,14 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) {
dError("failed to create notify thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create notify thread since %s", strerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -308,12 +312,14 @@ void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) {
dError("failed to create monitor thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create monitor thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -322,12 +328,14 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
dError("failed to create audit thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create audit thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -350,6 +358,7 @@ void dmStopAuditThread(SDnodeMgmt *pMgmt) {
}
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
if (!tsEnableCrashReport) {
return 0;
}
@ -358,8 +367,9 @@ int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) {
dError("failed to create crashReport thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create crashReport thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -431,8 +441,8 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg);
break;
default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg);
code = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue, reason:%s", pMsg, tstrerror(code));
break;
}

135
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -21,11 +21,11 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
int32_t code = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code);
if (code < 0) return -1;
if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code);
if (code < 0) return 0;
if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code);
if (code < 0) return 0;
if (code < 0) return code;
SJson *replicas = tjsonGetObjectItem(pJson, "replicas");
if (replicas == NULL) return 0;
@ -35,17 +35,17 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonGetArrayItem(replicas, i);
if (replica == NULL) return -1;
if (replica == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
SReplica *pReplica = pOption->replicas + i;
tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code);
if (code < 0) return -1;
if (code < 0) return code;
code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn);
if (code < 0) return -1;
if (code < 0) return code;
tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code);
if (code < 0) return -1;
if (code < 0) return code;
tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
if (code < 0) return -1;
if (code < 0) return code;
if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
pOption->numOfReplicas++;
}
@ -63,36 +63,41 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char *pData = NULL;
SJson *pJson = NULL;
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file);
dInfo("mnode file:%s not exist, reason:%s", file, tstrerror(TAOS_SYSTEM_ERROR(errno)));
return 0;
}
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
@ -100,12 +105,11 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
pJson = tjsonParse(pData);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (mmDecodeOption(pJson, pOption) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
if ((code = mmDecodeOption(pJson, pOption)) < 0) {
goto _OVER;
}
@ -118,37 +122,42 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read mnode file:%s since %s", file, terrstr());
dError("failed to read mnode file:%s since %s", file, tstrerror(code));
}
return code;
}
static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) {
int32_t code = 0;
if (pOption->deploy && pOption->numOfTotalReplicas > 0) {
if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1;
if ((code = tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex)) < 0) return code;
SJson *replicas = tjsonCreateArray();
if (replicas == NULL) return -1;
if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1;
if (replicas == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "replicas", replicas)) < 0) return code;
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonCreateObject();
if (replica == NULL) return -1;
if (replica == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
const SReplica *pReplica = pOption->replicas + i;
if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1;
if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1;
if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1;
if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1;
if (tjsonAddItemToArray(replicas, replica) < 0) return -1;
if ((code = tjsonAddDoubleToObject(replica, "id", pReplica->id)) < 0) return code;
if ((code = tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn)) < 0) return code;
if ((code = tjsonAddDoubleToObject(replica, "port", pReplica->port)) < 0) return code;
if ((code = tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i])) < 0) return code;
if ((code = tjsonAddItemToArray(replicas, replica)) < 0) return code;
}
}
if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1;
if ((code = tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex)) < 0) return code;
if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1;
if ((code = tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy)) < 0) return code;
return 0;
return code;
}
int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
@ -158,28 +167,59 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (mmEncodeOption(pJson, pOption) != 0) goto _OVER;
if (pJson == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
TAOS_CHECK_GOTO(mmEncodeOption(pJson, pOption), NULL, _OVER);
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosCloseFile(&pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy);
_OVER:
@ -188,8 +228,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, terrstr(), pOption->deploy);
dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, tstrerror(code), pOption->deploy);
}
return code;
}

37
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -26,11 +26,12 @@ void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) {
}
int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId;
SDCreateMnodeReq createReq = {0};
if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
SMnodeOpt option = {.deploy = true,
@ -56,43 +57,45 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
}
if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s, selfIndex is -1", terrstr());
return -1;
code = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s, selfIndex is -1", tstrerror(code));
return code;
}
if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
return -1;
if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", tstrerror(code));
return code;
}
return 0;
}
int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId;
SDDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dGError("failed to drop mnode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dGError("failed to drop mnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
SMnodeOpt option = {.deploy = false};
if (mmWriteFile(pInput->path, &option) != 0) {
dGError("failed to write mnode file since %s", terrstr());
if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&dropReq);
return 0;
return code;
}
SArray *mmGetMsgHandles() {

68
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -25,9 +25,10 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
}
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
int32_t code = 0;
SMnodeOpt option = {0};
if (mmReadFile(pInput->path, &option) != 0) {
return -1;
if ((code = mmReadFile(pInput->path, &option)) != 0) {
return code;
}
if (!option.deploy) {
@ -41,7 +42,7 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
dInfo("deploy mnode required. option deploy:%d", option.deploy);
}
return 0;
return code;
}
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
@ -73,22 +74,31 @@ static void mmClose(SMnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt);
}
static int32_t mndOpenWrapper(const char *path, SMnodeOpt *opt, SMnode **pMnode) {
int32_t code = 0;
*pMnode = mndOpen(path, opt);
if (*pMnode == NULL) {
code = terrno;
}
///*pMnode = pNode;
return code;
}
static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (walInit() != 0) {
dError("failed to init wal since %s", terrstr());
return -1;
int32_t code = 0;
if ((code = walInit()) != 0) {
dError("failed to init wal since %s", tstrerror(code));
return code;
}
if (syncInit() != 0) {
dError("failed to init sync since %s", terrstr());
return -1;
if ((code = syncInit()) != 0) {
dError("failed to init sync since %s", tstrerror(code));
return code;
}
SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pMgmt->pData = pInput->pData;
@ -100,10 +110,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
taosThreadRwlockInit(&pMgmt->lock, NULL);
SMnodeOpt option = {0};
if (mmReadFile(pMgmt->path, &option) != 0) {
dError("failed to read file since %s", terrstr());
if ((code = mmReadFile(pMgmt->path, &option)) != 0) {
dError("failed to read file since %s", tstrerror(code));
mmClose(pMgmt);
return -1;
return code;
}
if (!option.deploy) {
@ -115,18 +125,18 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
mmBuildOptionForOpen(pMgmt, &option);
}
pMgmt->pMnode = mndOpen(pMgmt->path, &option);
if (pMgmt->pMnode == NULL) {
dError("failed to open mnode since %s", terrstr());
code = mndOpenWrapper(pMgmt->path, &option, &pMgmt->pMnode);
if (code != 0) {
dError("failed to open mnode since %s", tstrerror(code));
mmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("mnode-impl", "initialized");
if (mmStartWorker(pMgmt) != 0) {
dError("failed to start mnode worker since %s", terrstr());
if ((code = mmStartWorker(pMgmt)) != 0) {
dError("failed to start mnode worker since %s", tstrerror(code));
mmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("mnode-worker", "initialized");
@ -134,9 +144,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
option.deploy = true;
option.numOfReplicas = 0;
option.numOfTotalReplicas = 0;
if (mmWriteFile(pMgmt->path, &option) != 0) {
dError("failed to write mnode file since %s", terrstr());
return -1;
if ((code = mmWriteFile(pMgmt->path, &option)) != 0) {
dError("failed to write mnode file since %s", tstrerror(code));
return code;
}
}
@ -160,13 +170,9 @@ static void mmStop(SMnodeMgmt *pMgmt) {
mndStop(pMgmt->pMnode);
}
static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) {
return mndIsCatchUp(pMgmt->pMnode);
}
static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { return mndIsCatchUp(pMgmt->pMnode); }
static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) {
return mndGetRole(pMgmt->pMnode);
}
static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { return mndGetRole(pMgmt->pMnode); }
SMgmtFunc mmGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0};

51
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -179,13 +179,15 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pWorker = &pMgmt->syncRdWorker;
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_PARA;
}
if (pWorker == NULL) return -1;
if (pWorker == NULL) return code;
SRpcMsg *pMsg;
code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) return code;
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL;
@ -201,6 +203,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
}
int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg qCfg = {
.min = tsNumOfMnodeQueryThreads,
.max = tsNumOfMnodeQueryThreads,
@ -209,9 +212,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.param = pMgmt,
.poolType = QUERY_AUTO_QWORKER_POOL,
};
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) {
dError("failed to start mnode-query worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &qCfg)) != 0) {
dError("failed to start mnode-query worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg fCfg = {
@ -221,9 +224,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg) != 0) {
dError("failed to start mnode-fetch worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg)) != 0) {
dError("failed to start mnode-fetch worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg rCfg = {
@ -233,9 +236,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) {
dError("failed to start mnode-read worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->readWorker, &rCfg)) != 0) {
dError("failed to start mnode-read worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg wCfg = {
@ -245,9 +248,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) {
dError("failed to start mnode-write worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->writeWorker, &wCfg)) != 0) {
dError("failed to start mnode-write worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg sCfg = {
@ -257,9 +260,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) {
dError("failed to start mnode mnode-sync worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->syncWorker, &sCfg)) != 0) {
dError("failed to start mnode mnode-sync worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg scCfg = {
@ -269,9 +272,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) {
dError("failed to start mnode mnode-sync-rd worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg)) != 0) {
dError("failed to start mnode mnode-sync-rd worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg arbCfg = {
@ -281,13 +284,13 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg,
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg) != 0) {
dError("failed to start mnode mnode-arb worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg)) != 0) {
dError("failed to start mnode mnode-arb worker since %s", tstrerror(code));
return code;
}
dDebug("mnode workers are initialized");
return 0;
return code;
}
void mmStopWorker(SMnodeMgmt *pMgmt) {

34
source/dnode/mgmt/mgmt_qnode/src/qmHandle.c
View File

@ -30,24 +30,25 @@ void qmGetQnodeLoads(SQnodeMgmt *pMgmt, SQnodeLoad *pInfo) {
}
int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create qnode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to create qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write qnode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&createReq);
@ -55,24 +56,25 @@ int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
}
int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to drop qnode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write qnode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&dropReq);

30
source/dnode/mgmt/mgmt_qnode/src/qmInt.c
View File

@ -33,11 +33,18 @@ static void qmClose(SQnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt);
}
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption);
if (*pQnode == NULL) {
return terrno;
}
return 0;
}
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
pMgmt->pData = pInput->pData;
@ -50,29 +57,30 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SQnodeOpt option = {0};
qmInitOption(pMgmt, &option);
pMgmt->pQnode = qndOpen(&option);
if (pMgmt->pQnode == NULL) {
dError("failed to open qnode since %s", terrstr());
code = qndOpenWrapper(&option, &pMgmt->pQnode);
if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt);
return -1;
}
tmsgReportStartup("qnode-impl", "initialized");
if (udfcOpen() != 0) {
if ((code = udfcOpen()) != 0) {
dError("qnode can not open udfc");
qmClose(pMgmt);
return -1;
return code;
}
if (qmStartWorker(pMgmt) != 0) {
dError("failed to start qnode worker since %s", terrstr());
if ((code = qmStartWorker(pMgmt)) != 0) {
dError("failed to start qnode worker since %s", tstrerror(code));
qmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("qnode-worker", "initialized");
pOutput->pMgmt = pMgmt;
return 0;
return code;
}
SMgmtFunc qmGetMgmtFunc() {

16
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -102,6 +102,8 @@ int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
}
int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg queryCfg = {
.min = tsNumOfVnodeQueryThreads,
.max = tsNumOfVnodeQueryThreads,
@ -111,9 +113,9 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.poolType = QUERY_AUTO_QWORKER_POOL,
};
if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) {
dError("failed to start qnode-query worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg)) != 0) {
dError("failed to start qnode-query worker since %s", tstrerror(code));
return code;
}
SSingleWorkerCfg fetchCfg = {
@ -124,13 +126,13 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg) != 0) {
dError("failed to start qnode-fetch worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg)) != 0) {
dError("failed to start qnode-fetch worker since %s", tstrerror(code));
return code;
}
dDebug("qnode workers are initialized");
return 0;
return code;
}
void qmStopWorker(SQnodeMgmt *pMgmt) {

35
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -19,24 +19,25 @@
void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {}
int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create snode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to create snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write snode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&createReq);
@ -44,24 +45,26 @@ int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
}
int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
return code;
}
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to drop snode since %s", terrstr());
code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) {
dError("failed to write snode file since %s", terrstr());
if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq);
return -1;
return code;
}
tFreeSMCreateQnodeReq(&dropReq);

38
source/dnode/mgmt/mgmt_snode/src/smInt.c
View File

@ -33,12 +33,19 @@ static void smClose(SSnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt);
}
int32_t sndOpenWrapper(const char *path, SSnodeOpt *pOption, SSnode **pNode) {
*pNode = sndOpen(path, pOption);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt));
if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pMgmt->pData = pInput->pData;
@ -50,35 +57,34 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SSnodeOpt option = {0};
smInitOption(pMgmt, &option);
pMgmt->pSnode = sndOpen(pMgmt->path, &option);
if (pMgmt->pSnode == NULL) {
dError("failed to open snode since %s", terrstr());
code = sndOpenWrapper(pMgmt->path, &option, &pMgmt->pSnode);
if (code != 0) {
dError("failed to open snode since %s", tstrerror(code));
smClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("snode-impl", "initialized");
if (smStartWorker(pMgmt) != 0) {
dError("failed to start snode worker since %s", terrstr());
if ((code = smStartWorker(pMgmt)) != 0) {
dError("failed to start snode worker since %s", tstrerror(code));
smClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("snode-worker", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in snode");
if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in snode since:%s", tstrerror(code));
smClose(pMgmt);
return -1;
return code;
}
pOutput->pMgmt = pMgmt;
return 0;
}
static int32_t smStartSnodes(SSnodeMgmt *pMgmt) {
return sndInit(pMgmt->pSnode);
}
static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { return sndInit(pMgmt->pSnode); }
SMgmtFunc smGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0};

60
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -68,17 +68,18 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
}
int32_t smStartWorker(SSnodeMgmt *pMgmt) {
int32_t code = 0;
pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->writeWroker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) {
SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pWriteWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
SMultiWorkerCfg cfg = {
@ -87,13 +88,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.fp = smProcessWriteQueue,
.param = pMgmt,
};
if (tMultiWorkerInit(pWriteWorker, &cfg) != 0) {
dError("failed to start snode-unique worker since %s", terrstr());
return -1;
if ((code = tMultiWorkerInit(pWriteWorker, &cfg)) != 0) {
dError("failed to start snode-unique worker since %s", tstrerror(code));
return code;
}
if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
@ -105,13 +106,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.param = pMgmt,
};
if (tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) {
dError("failed to start snode shared-worker since %s", terrstr());
return -1;
if ((code = tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) != 0) {
dError("failed to start snode shared-worker since %s", tstrerror(code));
return code;
}
dDebug("snode workers are initialized");
return 0;
return code;
}
void smStopWorker(SSnodeMgmt *pMgmt) {
@ -133,17 +134,18 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) {
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return code = terrno;
}
SSnode *pSnode = pMgmt->pSnode;
if (pSnode == NULL) {
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, terrstr(),
code = terrno;
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, tstrerror(code),
TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen);
taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return code;
}
SMsgHead *pHead = pRpc->pCont;
@ -154,48 +156,44 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
switch (qtype) {
case STREAM_QUEUE:
smPutNodeMsgToStreamQueue(pMgmt, pMsg);
code = smPutNodeMsgToStreamQueue(pMgmt, pMsg);
break;
case WRITE_QUEUE:
smPutNodeMsgToWriteQueue(pMgmt, pMsg);
code = smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break;
default:
terrno = TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_PARA;
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
return -1;
return code;
}
return 0;
return code;
}
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
return taosWriteQitem(pWorker->queue, pMsg);
}
int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
return taosWriteQitem(pWorker->queue, pMsg);
}
int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->streamWorker;
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg);
return 0;
return taosWriteQitem(pWorker->queue, pMsg);
}

42
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -54,24 +54,24 @@ typedef struct {
} SWrapperCfg;
typedef struct {
int32_t vgId;
int32_t vgVersion;
int32_t refCount;
int8_t dropped;
int8_t failed;
int8_t disable;
int32_t diskPrimary;
int32_t toVgId;
char *path;
SVnode *pImpl;
SMultiWorker pWriteW;
SMultiWorker pSyncW;
SMultiWorker pSyncRdW;
SMultiWorker pApplyW;
STaosQueue *pQueryQ;
STaosQueue *pStreamQ;
STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ;
int32_t vgId;
int32_t vgVersion;
int32_t refCount;
int8_t dropped;
int8_t failed;
int8_t disable;
int32_t diskPrimary;
int32_t toVgId;
char *path;
SVnode *pImpl;
SMultiWorker pWriteW;
SMultiWorker pSyncW;
SMultiWorker pSyncRdW;
SMultiWorker pApplyW;
STaosQueue *pQueryQ;
STaosQueue *pStreamQ;
STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ;
} SVnodeObj;
typedef struct {
@ -107,9 +107,9 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
// vmFile.c
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes);
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
// vmWorker.c
int32_t vmStartWorker(SVnodeMgmt *pMgmt);

143
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -19,12 +19,16 @@
#define MAX_CONTENT_LEN 2 * 1024 * 1024
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
taosThreadRwlockRdlock(&pMgmt->lock);
int32_t num = 0;
int32_t size = taosHashGetSize(pMgmt->hash);
SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
if (pVnodes == NULL) {
taosThreadRwlockUnlock(&pMgmt->lock);
return TSDB_CODE_OUT_OF_MEMORY;
}
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
@ -42,8 +46,9 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
taosThreadRwlockUnlock(&pMgmt->lock);
*numOfVnodes = num;
*ppVnodes = pVnodes;
return pVnodes;
return 0;
}
static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
@ -52,29 +57,32 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **
*ppCfgs = NULL;
SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes");
if (vnodes == NULL) return -1;
if (vnodes == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t vnodesNum = cJSON_GetArraySize(vnodes);
if (vnodesNum > 0) {
pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg));
if (pCfgs == NULL) return -1;
if (pCfgs == NULL) return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < vnodesNum; ++i) {
SJson *vnode = tjsonGetArrayItem(vnodes, i);
if (vnode == NULL) goto _OVER;
if (vnode == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
SWrapperCfg *pCfg = &pCfgs[i];
tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code);
if (code < 0) goto _OVER;
if (code != 0) goto _OVER;
snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
}
@ -98,33 +106,35 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file);
return 0;
code = TAOS_SYSTEM_ERROR(errno);
dInfo("vnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0;
return code;
}
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open vnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open vnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read vnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read vnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
@ -132,12 +142,12 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
pJson = tjsonParse(pData);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
@ -150,28 +160,36 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read vnode file:%s since %s", file, terrstr());
dError("failed to read vnode file:%s since %s", file, tstrerror(code));
}
return code;
}
static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) {
SJson *vnodes = tjsonCreateArray();
if (vnodes == NULL) return -1;
if (tjsonAddItemToObject(pJson, "vnodes", vnodes) < 0) return -1;
int32_t code = 0;
SJson *vnodes = tjsonCreateArray();
if (vnodes == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "vnodes", vnodes)) < 0) {
tjsonDelete(vnodes);
return code;
};
for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i];
if (pVnode == NULL) continue;
SJson *vnode = tjsonCreateObject();
if (vnode == NULL) return -1;
if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1;
if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1;
if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1;
if (tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary) < 0) return -1;
if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1;
if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1;
if (vnode == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if ((code = tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId)) < 0) return code;
if ((code = tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped)) < 0) return code;
if ((code = tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion)) < 0) return code;
if ((code = tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary)) < 0) return code;
if (pVnode->toVgId) {
if ((code = tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId)) < 0) return code;
}
if ((code = tjsonAddItemToArray(vnodes, vnode)) < 0) return code;
}
return 0;
@ -185,30 +203,60 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
int32_t nBytes = snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
return TSDB_CODE_OUT_OF_RANGE;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t numOfVnodes = 0;
ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
if (ppVnodes == NULL) goto _OVER;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code) goto _OVER;
terrno = TSDB_CODE_OUT_OF_MEMORY;
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes) != 0) goto _OVER;
if (pJson == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER;
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
code = taosCloseFile(&pFile);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes);
@ -228,8 +276,7 @@ _OVER:
}
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes);
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, tstrerror(code), numOfVnodes);
}
return code;
}

33
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -256,8 +256,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
char path[TSDB_FILENAME_LEN] = {0};
if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
if (req.learnerReplica == 0) {
@ -298,25 +297,24 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort ||
strcmp(pReplica->fqdn, tsLocalFqdn) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", req.vgId, pReplica->id, pReplica->fqdn,
pReplica->port);
return -1;
code = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode, reason:%s", req.vgId, pReplica->id,
pReplica->fqdn, pReplica->port, tstrerror(code));
return code;
}
if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId);
return -1;
code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty, reason:%s", req.vgId, tstrerror(code));
return code;
}
}
vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr());
code = terrno != 0 ? terrno : -1;
if ((code = vmTsmaAdjustDays(&vnodeCfg, &req)) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, tstrerror(code));
goto _OVER;
}
@ -327,8 +325,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_VND_ALREADY_EXIST;
code = terrno;
code = TSDB_CODE_VND_ALREADY_EXIST;
return 0;
}
@ -377,14 +374,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _OVER;
}
taosThreadMutexLock(&pMgmt->createLock);
//taosThreadMutexLock(&pMgmt->createLock);
code = vmWriteVnodeListToFile(pMgmt);
if (code != 0) {
code = terrno != 0 ? terrno : code;
taosThreadMutexUnlock(&pMgmt->createLock);
//taosThreadMutexUnlock(&pMgmt->createLock);
goto _OVER;
}
taosThreadMutexUnlock(&pMgmt->createLock);
//taosThreadMutexUnlock(&pMgmt->createLock);
_OVER:
if (code != 0) {
@ -988,7 +985,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMultiMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

101
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -33,6 +33,7 @@ int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
}
int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t code = 0;
STfs *pTfs = pMgmt->pTfs;
int32_t diskId = 0;
if (!pTfs) {
@ -59,7 +60,12 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
// alloc
int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0};
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
return code;
}
for (int32_t v = 0; v < numOfVnodes; v++) {
SVnodeObj *pVnode = ppVnodes[v];
disks[pVnode->diskPrimary] += 1;
@ -436,6 +442,7 @@ static void *vmCloseVnodeInThread(void *param) {
}
static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
dInfo("start to close all vnodes");
tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dInfo("vnodes mgmt worker is stopped");
@ -443,7 +450,12 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
dInfo("vnodes multiple mgmt worker is stopped");
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1;
@ -513,8 +525,14 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
}
static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
if (ppVnodes != NULL) {
for (int32_t i = 0; i < numOfVnodes; ++i) {
@ -549,12 +567,14 @@ static void *vmThreadFp(void *param) {
}
static int32_t vmInitTimer(SVnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) {
dError("failed to create vnode timer thread since %s", strerror(errno));
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create vnode timer thread since %s", tstrerror(code));
return code;
}
taosThreadAttrDestroy(&thAttr);
@ -573,7 +593,10 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = -1;
SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt));
if (pMgmt == NULL) goto _OVER;
if (pMgmt == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pMgmt->pData = pInput->pData;
pMgmt->path = pInput->path;
@ -582,8 +605,18 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue;
pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize;
pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
taosThreadMutexInit(&pMgmt->createLock, NULL);
code = taosThreadRwlockInit(&pMgmt->lock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = taosThreadMutexInit(&pMgmt->createLock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
pMgmt->pTfs = pInput->pTfs;
if (pMgmt->pTfs == NULL) {
@ -592,38 +625,39 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
}
tmsgReportStartup("vnode-tfs", "initialized");
if (walInit() != 0) {
dError("failed to init wal since %s", terrstr());
if ((code = walInit()) != 0) {
dError("failed to init wal since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-wal", "initialized");
if (syncInit() != 0) {
dError("failed to open sync since %s", terrstr());
if ((code = syncInit()) != 0) {
dError("failed to open sync since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-sync", "initialized");
if (vnodeInit(tsNumOfCommitThreads) != 0) {
dError("failed to init vnode since %s", terrstr());
if ((code = vnodeInit(tsNumOfCommitThreads)) != 0) {
dError("failed to init vnode since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-commit", "initialized");
if (vmStartWorker(pMgmt) != 0) {
dError("failed to init workers since %s", terrstr());
if ((code = vmStartWorker(pMgmt)) != 0) {
dError("failed to init workers since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-worker", "initialized");
if (vmOpenVnodes(pMgmt) != 0) {
dError("failed to open all vnodes since %s", terrstr());
if ((code = vmOpenVnodes(pMgmt)) != 0) {
dError("failed to open all vnodes since %s", tstrerror(code));
goto _OVER;
}
tmsgReportStartup("vnode-vnodes", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in vnode");
if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in vnode since %s", tstrerror(code));
goto _OVER;
}
@ -633,7 +667,7 @@ _OVER:
if (code == 0) {
pOutput->pMgmt = pMgmt;
} else {
dError("failed to init vnodes-mgmt since %s", terrstr());
dError("failed to init vnodes-mgmt since %s", tstrerror(code));
vmCleanup(pMgmt);
}
@ -683,18 +717,32 @@ static void *vmRestoreVnodeInThread(void *param) {
}
static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return code;
}
int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread));
if (threads == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t;
threads[t].pMgmt = pMgmt;
threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *));
if (threads[t].ppVnodes == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
}
for (int32_t v = 0; v < numOfVnodes; ++v) {
@ -717,6 +765,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) {
dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno));
ASSERT(errno == 0);
}
taosThreadAttrDestroy(&thAttr);
@ -742,6 +791,14 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
}
return vmInitTimer(pMgmt);
_exit:
for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t];
taosMemoryFree(pThread->ppVnodes);
}
taosMemoryFree(threads);
return code;
}
static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); }

70
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -200,26 +200,33 @@ static bool vmDataSpaceSufficient(SVnodeObj *pVnode) {
}
}
static int32_t vmAcquireVnodeWrapper(SVnodeMgmt *pMgt, int32_t vgId, SVnodeObj **pNode) {
*pNode = vmAcquireVnode(pMgt, vgId);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId;
if (pMsg->contLen < sizeof(SMsgHead)) {
dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType),
pMsg->contLen);
return -1;
return TSDB_CODE_INVALID_MSG;
}
SMsgHead *pHead = pMsg->pCont;
int32_t code = 0;
pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) {
SVnodeObj *pVnode = NULL;
code = vmAcquireVnodeWrapper(pMgmt, pHead->vgId, &pVnode);
if (code != 0) {
dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
terrno = (terrno != 0) ? terrno : -1;
return terrno;
tstrerror(code), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
return code;
}
switch (qtype) {
@ -234,49 +241,45 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
break;
case STREAM_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pStreamQ, pMsg);
code = taosWriteQitem(pVnode->pStreamQ, pMsg);
break;
case FETCH_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pFetchQ, pMsg);
code = taosWriteQitem(pVnode->pFetchQ, pMsg);
break;
case WRITE_QUEUE:
if (!vmDataSpaceSufficient(pVnode)) {
terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
code = terrno;
code = TSDB_CODE_NO_ENOUGH_DISKSPACE;
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break;
}
if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) {
terrno = TSDB_CODE_VND_NO_WRITE_AUTH;
code = terrno;
code = TSDB_CODE_VND_NO_WRITE_AUTH;
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break;
}
if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) {
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg);
terrno = TSDB_CODE_VND_STOPPED;
code = terrno;
code = TSDB_CODE_VND_STOPPED;
break;
}
dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pWriteW.queue, pMsg);
code = taosWriteQitem(pVnode->pWriteW.queue, pMsg);
break;
case SYNC_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncW.queue, pMsg);
code = taosWriteQitem(pVnode->pSyncW.queue, pMsg);
break;
case SYNC_RD_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
code = taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
break;
case APPLY_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pApplyW.queue, pMsg);
code = taosWriteQitem(pVnode->pApplyW.queue, pMsg);
break;
default:
code = -1;
terrno = TSDB_CODE_INVALID_PARA;
code = TSDB_CODE_INVALID_MSG;
break;
}
@ -299,15 +302,13 @@ int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMs
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
return 0;
return taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
}
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg);
return 0;
return taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg);
}
int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
@ -317,7 +318,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pRpc->contLen);
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return TSDB_CODE_INVALID_MSG;
}
EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM;
@ -326,7 +327,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) {
rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL;
return -1;
return code;
}
SMsgHead *pHead = pRpc->pCont;
@ -383,6 +384,7 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
}
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
int32_t code = 0;
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
@ -398,8 +400,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL ||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
@ -428,26 +429,27 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
}
int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
int32_t code = 0;
SQueryAutoQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query";
pQPool->min = tsNumOfVnodeQueryThreads;
pQPool->max = tsNumOfVnodeQueryThreads;
if (tQueryAutoQWorkerInit(pQPool) != 0) return -1;
if ((code = tQueryAutoQWorkerInit(pQPool)) != 0) return code;
SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool;
pStreamPool->name = "vnode-stream";
pStreamPool->ratio = tsRatioOfVnodeStreamThreads;
if (tAutoQWorkerInit(pStreamPool) != 0) return -1;
if ((code = tAutoQWorkerInit(pStreamPool)) != 0) return code;
SWWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch";
pFPool->max = tsNumOfVnodeFetchThreads;
if (tWWorkerInit(pFPool) != 0) return -1;
if ((code = tWWorkerInit(pFPool)) != 0) return code;
SSingleWorkerCfg mgmtCfg = {
.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1;
if ((code = tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg)) != 0) return code;
int32_t threadNum = 0;
if (tsNumOfCores == 1) {
@ -461,7 +463,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
.fp = (FItem)vmProcessMultiMgmtQueue,
.param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg) != 0) return -1;
if ((code = tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg)) != 0) return code;
dDebug("vnode workers are initialized");
return 0;

1
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -98,6 +98,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType);
int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode);
int32_t dmInitVarsWrapper(SDnode *pDnode);
void dmClearVars(SDnode *pDnode);
int32_t dmInitModule(SDnode *pDnode);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);

157
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -14,20 +14,22 @@
*/
#define _DEFAULT_SOURCE
// clang-format off
#include "dmMgmt.h"
#include "audit.h"
#include "libs/function/tudf.h"
#include "tgrant.h"
#include "tcompare.h"
// clang-format on
#define DM_INIT_AUDIT() \
do { \
auditCfg.port = tsMonitorPort; \
auditCfg.server = tsMonitorFqdn; \
auditCfg.comp = tsMonitorComp; \
if (auditInit(&auditCfg) != 0) { \
return -1; \
} \
#define DM_INIT_AUDIT() \
do { \
auditCfg.port = tsMonitorPort; \
auditCfg.server = tsMonitorFqdn; \
auditCfg.comp = tsMonitorComp; \
if ((code = auditInit(&auditCfg)) != 0) { \
return code; \
} \
} while (0)
static SDnode globalDnode = {0};
@ -35,10 +37,11 @@ static SDnode globalDnode = {0};
SDnode *dmInstance() { return &globalDnode; }
static int32_t dmCheckRepeatInit(SDnode *pDnode) {
int32_t code = 0;
if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
dError("env is already initialized");
terrno = TSDB_CODE_REPEAT_INIT;
return -1;
code = TSDB_CODE_REPEAT_INIT;
return code;
}
return 0;
}
@ -58,18 +61,15 @@ static int32_t dmInitMonitor() {
monCfg.port = tsMonitorPort;
monCfg.server = tsMonitorFqdn;
monCfg.comp = tsMonitorComp;
if (monInit(&monCfg) != 0) {
if (terrno != 0) code = terrno;
goto _exit;
if ((code = monInit(&monCfg)) != 0) {
dError("failed to init monitor since %s", tstrerror(code));
}
_exit:
if (code) terrno = code;
return code;
}
static int32_t dmInitAudit() {
SAuditCfg auditCfg = {0};
int32_t code = 0;
DM_INIT_AUDIT();
@ -88,29 +88,35 @@ static bool dmDataSpaceAvailable() {
return true;
}
static bool dmCheckDiskSpace() {
static int32_t dmCheckDiskSpace() {
osUpdate();
// availability
bool ret = true;
int32_t code = 0;
if (!dmDataSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
ret = false;
code = TSDB_CODE_NO_DISKSPACE;
return code;
}
if (!osLogSpaceAvailable()) {
dError("log disk space unavailable, i.e. %s", tsLogDir);
terrno = TSDB_CODE_NO_DISKSPACE;
ret = false;
code = TSDB_CODE_NO_DISKSPACE;
return code;
}
if (!osTempSpaceAvailable()) {
dError("temp disk space unavailable, i.e. %s", tsTempDir);
terrno = TSDB_CODE_NO_DISKSPACE;
ret = false;
code = TSDB_CODE_NO_DISKSPACE;
return code;
}
return ret;
return code;
}
int32_t tfsOpenWrapper(SDiskCfg *pCfg, int32_t ndisk, STfs **tfs) {
*tfs = tfsOpen(pCfg, ndisk);
if (*tfs == NULL) {
return terrno;
}
return 0;
}
int32_t dmDiskInit() {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
tstrncpy(dCfg.dir, tsDataDir, TSDB_FILENAME_LEN);
@ -121,8 +127,8 @@ int32_t dmDiskInit() {
numOfDisks = 1;
}
code = tfsOpen(pDisks, numOfDisks, &pDnode->pTfs);
if (pDnode->pTfs == NULL) {
int32_t code = tfsOpenWrapper(pDisks, numOfDisks, &pDnode->pTfs);
if (code != 0) {
dError("failed to init tfs since %s", tstrerror(code));
TAOS_RETURN(code);
}
@ -147,6 +153,12 @@ static bool dmCheckDataDirVersion() {
return true;
}
static int32_t dmCheckDataDirVersionWrapper() {
if (!dmCheckDataDirVersion()) {
return TSDB_CODE_INVALID_DATA_FMT;
}
return 0;
}
#if defined(USE_S3)
extern int32_t s3Begin();
@ -157,17 +169,21 @@ extern int8_t tsS3Enabled;
int32_t dmInit() {
dInfo("start to init dnode env");
if (dmDiskInit() != 0) return -1;
if (!dmCheckDataDirVersion()) return -1;
if (!dmCheckDiskSpace()) return -1;
if (dmCheckRepeatInit(dmInstance()) != 0) return -1;
if (dmInitSystem() != 0) return -1;
if (dmInitMonitor() != 0) return -1;
if (dmInitAudit() != 0) return -1;
if (dmInitDnode(dmInstance()) != 0) return -1;
if (InitRegexCache() != 0) return -1;
int32_t code = 0;
if ((code = dmDiskInit()) != 0) return code;
if (!dmCheckDataDirVersion()) {
code = TSDB_CODE_INVALID_DATA_FMT;
return code;
}
if ((code = dmCheckDiskSpace()) != 0) return code;
if ((code = dmCheckRepeatInit(dmInstance())) != 0) return code;
if ((code = dmInitSystem()) != 0) return code;
if ((code = dmInitMonitor()) != 0) return code;
if ((code = dmInitAudit()) != 0) return code;
if ((code = dmInitDnode(dmInstance())) != 0) return code;
if ((code = InitRegexCache() != 0)) return code;
#if defined(USE_S3)
if (s3Begin() != 0) return -1;
if ((code = s3Begin()) != 0) return code;
#endif
dInfo("dnode env is initialized");
@ -218,6 +234,7 @@ int32_t dmRun() {
}
static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
@ -225,19 +242,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dmReleaseWrapper(pWrapper);
switch (ntype) {
case MNODE:
terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
code = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
break;
case QNODE:
terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
code = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
break;
case SNODE:
terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
code = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
break;
default:
terrno = TSDB_CODE_APP_ERROR;
code = TSDB_CODE_APP_ERROR;
}
dError("failed to create node since %s", terrstr());
return -1;
dError("failed to create node since %s", tstrerror(code));
return code;
}
dInfo("start to process create-node-request");
@ -245,18 +262,18 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper);
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
}
taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg);
code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else {
dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper);
@ -272,12 +289,13 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
}
static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) {
dError("fail to process alter node type since node not exist");
return -1;
return TSDB_CODE_INVALID_MSG;
}
dmReleaseWrapper(pWrapper);
@ -290,16 +308,16 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dInfo("node:%s, checking node role:%d", pWrapper->name, role);
if (role == TAOS_SYNC_ROLE_VOTER) {
dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role);
terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
return -1;
code = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
return code;
}
}
if (pWrapper->func.isCatchUpFp != NULL) {
dInfo("node:%s, checking node catch up", pWrapper->name);
if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) {
terrno = TSDB_CODE_MNODE_NOT_CATCH_UP;
return -1;
code = TSDB_CODE_MNODE_NOT_CATCH_UP;
return code;
}
}
@ -315,17 +333,17 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex);
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
}
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg);
code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else {
dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper);
@ -341,35 +359,36 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
}
static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) {
switch (ntype) {
case MNODE:
terrno = TSDB_CODE_MNODE_NOT_DEPLOYED;
code = TSDB_CODE_MNODE_NOT_DEPLOYED;
break;
case QNODE:
terrno = TSDB_CODE_QNODE_NOT_DEPLOYED;
code = TSDB_CODE_QNODE_NOT_DEPLOYED;
break;
case SNODE:
terrno = TSDB_CODE_SNODE_NOT_DEPLOYED;
code = TSDB_CODE_SNODE_NOT_DEPLOYED;
break;
default:
terrno = TSDB_CODE_APP_ERROR;
code = TSDB_CODE_APP_ERROR;
}
dError("failed to drop node since %s", terrstr());
return -1;
dError("failed to drop node since %s", tstrerror(code));
return terrno = code;
}
taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to drop", pWrapper->name);
int32_t code = (*pWrapper->func.dropFp)(&input, pMsg);
code = (*pWrapper->func.dropFp)(&input, pMsg);
if (code != 0) {
dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
dError("node:%s, failed to drop since %s", pWrapper->name, tstrerror(code));
} else {
dInfo("node:%s, has been dropped", pWrapper->name);
pWrapper->required = false;
@ -417,6 +436,4 @@ void dmReportStartup(const char *pName, const char *pDesc) {
int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
bool dmReadyForTest() {
return dmInstance()->data.dnodeVer > 0;
}
bool dmReadyForTest() { return dmInstance()->data.dnodeVer > 0; }

54
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -42,7 +42,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
int32_t code = -1;
char path[PATH_MAX + 100] = {0};
if (dmInitVars(pDnode) != 0) {
if ((code = dmInitVarsWrapper(pDnode)) != 0) {
goto _OVER;
}
@ -65,26 +65,31 @@ int32_t dmInitDnode(SDnode *pDnode) {
snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = taosStrdup(path);
if (pWrapper->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pWrapper->required = dmRequireNode(pDnode, pWrapper);
}
pDnode->lockfile = dmCheckRunning(tsDataDir);
if (pDnode->lockfile == NULL) {
code = dmCheckRunning(tsDataDir, &pDnode->lockfile);
if (code != 0) {
goto _OVER;
}
if (dmInitModule(pDnode) != 0) {
if ((code = dmInitModule(pDnode)) != 0) {
goto _OVER;
}
indexInit(tsNumOfCommitThreads);
streamMetaInit();
dmInitStatusClient(pDnode);
dmInitSyncClient(pDnode);
if ((code = dmInitStatusClient(pDnode)) != 0) {
goto _OVER;
}
if ((code = dmInitSyncClient(pDnode)) != 0) {
goto _OVER;
}
dmReportStartup("dnode-transport", "initialized");
dDebug("dnode is created, ptr:%p", pDnode);
@ -94,7 +99,7 @@ _OVER:
if (code != 0 && pDnode != NULL) {
dmClearVars(pDnode);
pDnode = NULL;
dError("failed to create dnode since %s", terrstr());
dError("failed to create dnode since %s", tstrerror(code));
}
return code;
@ -122,7 +127,15 @@ void dmCleanupDnode(SDnode *pDnode) {
dDebug("dnode is closed, ptr:%p", pDnode);
}
int32_t dmInitVarsWrapper(SDnode *pDnode) {
int32_t code = dmInitVars(pDnode);
if (code == -1) {
return terrno;
}
return 0;
}
int32_t dmInitVars(SDnode *pDnode) {
int32_t code = 0;
SDnodeData *pData = &pDnode->data;
pData->dnodeId = 0;
pData->clusterId = 0;
@ -138,21 +151,21 @@ int32_t dmInitVars(SDnode *pDnode) {
taosMemoryFreeClear(machineId);
} else {
#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
return -1;
code = TSDB_CODE_DNODE_NO_MACHINE_CODE;
return terrno = code;
#endif
}
pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pData->dnodeHash == NULL) {
dError("failed to init dnode hash");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
return terrno = code;
}
if (dmReadEps(pData) != 0) {
dError("failed to read file since %s", terrstr());
return -1;
if ((code = dmReadEps(pData)) != 0) {
dError("failed to read file since %s", tstrerror(code));
return code;
}
#if defined(TD_ENTERPRISE)
@ -274,22 +287,21 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
} else {
switch (pWrapper->ntype) {
case MNODE:
terrno = TSDB_CODE_MNODE_NOT_FOUND;
code = TSDB_CODE_MNODE_NOT_FOUND;
break;
case QNODE:
terrno = TSDB_CODE_QNODE_NOT_FOUND;
code = TSDB_CODE_QNODE_NOT_FOUND;
break;
case SNODE:
terrno = TSDB_CODE_SNODE_NOT_FOUND;
code = TSDB_CODE_SNODE_NOT_FOUND;
break;
case VNODE:
terrno = TSDB_CODE_VND_STOPPED;
code = TSDB_CODE_VND_STOPPED;
break;
default:
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
break;
}
code = -1;
}
taosThreadRwlockUnlock(&pWrapper->lock);

47
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -17,12 +17,13 @@
#include "dmMgmt.h"
int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
SDnode *pDnode = pWrapper->pDnode;
if (taosMkDir(pWrapper->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code));
return code;
}
SMgmtOutputOpt output = {0};
@ -30,9 +31,9 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to open", pWrapper->name);
tmsgSetDefault(&input.msgCb);
if ((*pWrapper->func.openFp)(&input, &output) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
return -1;
if ((code = (*pWrapper->func.openFp)(&input, &output)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return code;
}
dInfo("node:%s, has been opened", pWrapper->name);
pWrapper->deployed = true;
@ -46,11 +47,12 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
}
int32_t dmStartNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
if (pWrapper->func.startFp != NULL) {
dDebug("node:%s, start to start", pWrapper->name);
if ((*pWrapper->func.startFp)(pWrapper->pMgmt) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
return -1;
if ((code = (*pWrapper->func.startFp)(pWrapper->pMgmt)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return code;
}
dDebug("node:%s, has been started", pWrapper->name);
}
@ -86,12 +88,13 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
}
static int32_t dmOpenNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue;
if (dmOpenNode(pWrapper) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
return -1;
if ((code = dmOpenNode(pWrapper)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return code;
}
}
@ -100,12 +103,13 @@ static int32_t dmOpenNodes(SDnode *pDnode) {
}
static int32_t dmStartNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue;
if (dmStartNode(pWrapper) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
return -1;
if ((code = dmStartNode(pWrapper)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return code;
}
}
@ -129,19 +133,20 @@ static void dmCloseNodes(SDnode *pDnode) {
}
int32_t dmRunDnode(SDnode *pDnode) {
int32_t code = 0;
int32_t count = 0;
if (dmOpenNodes(pDnode) != 0) {
dError("failed to open nodes since %s", terrstr());
if ((code = dmOpenNodes(pDnode)) != 0) {
dError("failed to open nodes since %s", tstrerror(code));
dmCloseNodes(pDnode);
return -1;
return code;
}
if (dmStartNodes(pDnode) != 0) {
dError("failed to start nodes since %s", terrstr());
if ((code = dmStartNodes(pDnode)) != 0) {
dError("failed to start nodes since %s", tstrerror(code));
dmSetStatus(pDnode, DND_STAT_STOPPED);
dmStopNodes(pDnode);
dmCloseNodes(pDnode);
return -1;
return code;
}
while (1) {

68
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -39,9 +39,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
if (msgFp == NULL) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
// terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
return -1;
return TSDB_CODE_MSG_NOT_PROCESSED;
}
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
@ -54,14 +54,15 @@ static bool dmFailFastFp(tmsg_t msgType) {
return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES;
}
static void dmConvertErrCode(tmsg_t msgType) {
if (terrno != TSDB_CODE_APP_IS_STOPPING) {
return;
static int32_t dmConvertErrCode(tmsg_t msgType, int32_t code) {
if (code != TSDB_CODE_APP_IS_STOPPING) {
return code;
}
if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) ||
(msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) {
terrno = TSDB_CODE_VND_STOPPED;
code = TSDB_CODE_VND_STOPPED;
}
return code;
}
static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite));
@ -99,14 +100,14 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
int32_t svrVer = 0;
taosVersionStrToInt(version, &svrVer);
if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) {
if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) {
dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer);
goto _OVER;
}
bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp);
if (isForbidden) {
terrno = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
code = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
goto _OVER;
}
@ -119,7 +120,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
case TDMT_SCH_FETCH_RSP:
case TDMT_SCH_MERGE_FETCH_RSP:
case TDMT_VND_SUBMIT_RSP:
qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
code = qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
return;
case TDMT_MND_STATUS_RSP:
if (pEpSet != NULL) {
@ -148,32 +149,32 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
return;
} else {
if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
} else {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
}
goto _OVER;
}
}
} else {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
goto _OVER;
}
if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {
dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN;
code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
} else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) &&
(!IsReq(pRpc)) && (pRpc->pCont == NULL)) {
dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code));
terrno = pRpc->code;
code = pRpc->code;
goto _OVER;
}
if (pHandle->defaultNtype == NODE_END) {
dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
code = TSDB_CODE_MSG_NOT_PROCESSED;
goto _OVER;
}
@ -197,12 +198,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
}
} else {
dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN;
code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
}
}
if (dmMarkWrapper(pWrapper) != 0) {
if ((code = dmMarkWrapper(pWrapper)) != 0) {
pWrapper = NULL;
goto _OVER;
}
@ -221,12 +222,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
_OVER:
if (code != 0) {
dmConvertErrCode(pRpc->msgType);
if (terrno != 0) code = terrno;
code = dmConvertErrCode(pRpc->msgType, code);
if (pMsg) {
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code));
} else {
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr());
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, tstrerror(code));
}
if (IsReq(pRpc)) {
@ -280,17 +280,19 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
}
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
} else {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
}
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1;
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
pMsg->info.handle);
return code;
} else {
pMsg->info.handle = 0;
rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL);
@ -298,17 +300,19 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
}
}
static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING;
code = TSDB_CODE_APP_IS_STARTING;
} else {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = TSDB_CODE_APP_IS_STOPPING;
}
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle);
return -1;
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
pMsg->info.handle);
return code;
} else {
rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL);
return 0;
@ -426,7 +430,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
pTrans->statusRpc = rpcOpen(&rpcInit);
if (pTrans->statusRpc == NULL) {
dError("failed to init dnode rpc status client");
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
dDebug("dnode rpc status client is initialized");
@ -471,7 +475,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
pTrans->syncRpc = rpcOpen(&rpcInit);
if (pTrans->syncRpc == NULL) {
dError("failed to init dnode rpc sync client");
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
dDebug("dnode rpc sync client is initialized");
@ -521,7 +525,7 @@ int32_t dmInitServer(SDnode *pDnode) {
pTrans->serverRpc = rpcOpen(&rpcInit);
if (pTrans->serverRpc == NULL) {
dError("failed to init dnode rpc server");
return -1;
return terrno;
}
dDebug("dnode rpc server is initialized");

33
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -145,20 +145,20 @@ typedef struct {
} SDnodeData;
typedef struct {
const char *path;
const char *name;
STfs *pTfs;
SDnodeData *pData;
SMsgCb msgCb;
ProcessCreateNodeFp processCreateNodeFp;
const char *path;
const char *name;
STfs *pTfs;
SDnodeData *pData;
SMsgCb msgCb;
ProcessCreateNodeFp processCreateNodeFp;
ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp;
} SMgmtInputOpt;
typedef struct {
@ -203,9 +203,10 @@ void *dmSetMgmtHandle(SArray *pArray, tmsg_t msgType, void *nodeMsgFp, boo
void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
// dmFile.c
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed);
TdFilePtr dmCheckRunning(const char *dataDir);
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed);
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData);

122
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -95,18 +95,18 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
}
int dmOccurrences(char *str, char *toSearch) {
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
int count = 0;
char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++;
ptr++;
}
return count;
}
void dmSplitStr(char** arr, char* str, const char* del) {
void dmSplitStr(char **arr, char *str, const char *del) {
char *lasts;
char* s = strsep(&str, del);
char *s = strsep(&str, del);
while (s != NULL) {
*arr++ = s;
s = strsep(&str, del);
@ -132,11 +132,10 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) {
if (strlen(tsEncryptAlgorithm) > 0) {
if (strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4;
}
else{
} else {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER;
@ -144,45 +143,45 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){
if (scopeLen == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER;
}
char* tmp = taosMemoryMalloc(scopeLen + 1);
char *tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1));
char **array = taosMemoryMalloc(sizeof(char *) * (count + 1));
memset(array, 0, sizeof(char *) * (count + 1));
dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){
char* str = *(array + i);
for (int32_t i = 0; i < count + 1; i++) {
char *str = *(array + i);
bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_TSDB;
success = true;
}
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_VNODE_WAL;
success = true;
}
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_SDB;
success = true;
}
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){
if (strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_MNODE_WAL;
success = true;
}
if(!success){
if (!success) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp);
taosMemoryFree(array);
@ -196,7 +195,7 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope);
}
#endif
code = 0;
goto _OVER;
@ -204,26 +203,26 @@ int32_t dmReadEps(SDnodeData *pData) {
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
@ -232,12 +231,11 @@ int32_t dmReadEps(SDnodeData *pData) {
pJson = tjsonParse(content);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (dmDecodeEps(pJson, pData) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
if ((code = dmDecodeEps(pJson, pData)) < 0) {
goto _OVER;
}
@ -251,7 +249,7 @@ _OVER:
if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr());
return code;
return terrno = code;
}
if (taosArrayGetSize(pData->dnodeEps) == 0) {
@ -261,8 +259,8 @@ _OVER:
taosArrayPush(pData->dnodeEps, &dnodeEp);
}
if (dmReadDnodePairs(pData) != 0) {
return -1;
if ((code = dmReadDnodePairs(pData)) != 0) {
return terrno = code;
}
dDebug("reset dnode list on startup");
@ -270,8 +268,8 @@ _OVER:
if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) {
dError("localEp %s different with %s and need to be reconfigured", tsLocalEp, file);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
code = TSDB_CODE_INVALID_CFG;
return terrno = code;
}
return code;
@ -308,7 +306,7 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
}
int32_t dmWriteEps(SDnodeData *pData) {
int32_t code = -1;
int32_t code = 0;
char *buffer = NULL;
SJson *pJson = NULL;
TdFilePtr pFile = NULL;
@ -317,26 +315,30 @@ int32_t dmWriteEps(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY;
// if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmInitDndInfo(pData), NULL, _OVER);
if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
pData->engineVer = tsVersion;
if (dmEncodeEps(pJson, pData) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmEncodeEps(pJson, pData), NULL, _OVER); // dmEncodeEps(pJson, pData) != 0) goto _OVER;
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
code = 0;
pData->updateTime = taosGetTimestampMs();
@ -349,8 +351,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer);
dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, tstrerror(code), pData->dnodeVer);
}
return code;
}
@ -592,28 +593,29 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file);
code = TAOS_SYSTEM_ERROR(errno);
dDebug("dnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -627,12 +629,13 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
pJson = tjsonParse(content);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair));
if (pData->oldDnodeEps == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to calloc dnodeEp array since %s", strerror(errno));
goto _OVER;
}
@ -640,7 +643,8 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
if (dmDecodeEpPairs(pJson, pData) < 0) {
taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
@ -653,7 +657,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr());
dError("failed to read dnode file:%s since %s", file, tstrerror(code));
return code;
}
@ -680,8 +684,8 @@ _OVER:
pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id);
taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_CFG;
return -1;
code = TSDB_CODE_INVALID_CFG;
return code;
}
}
}

346
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -14,23 +14,23 @@
*/
#define _DEFAULT_SOURCE
#include "dmUtil.h"
#include "tjson.h"
#include "tgrant.h"
#include "crypt.h"
#include "dmUtil.h"
#include "tchecksum.h"
#include "tgrant.h"
#include "tjson.h"
#define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
#define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin"
static int32_t dmDecodeFile(SJson *pJson, bool *deployed) {
int32_t code = 0;
int32_t value = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code);
if (code < 0) return -1;
if (code < 0) return code;
*deployed = (value != 0);
return code;
@ -42,7 +42,11 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char *content = NULL;
SJson *pJson = NULL;
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file);
@ -52,27 +56,27 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read file:%s since %s", file, tstrerror(code));
goto _OVER;
}
@ -80,12 +84,12 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pJson = tjsonParse(content);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (dmDecodeFile(pJson, pDeployed) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
@ -98,13 +102,13 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr());
dError("failed to read dnode file:%s since %s", file, tstrerror(code));
}
return code;
}
static int32_t dmEncodeFile(SJson *pJson, bool deployed) {
if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
return 0;
}
@ -115,26 +119,57 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) {
TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (dmEncodeFile(pJson, deployed) != 0) goto _OVER;
if (pJson == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = dmEncodeFile(pJson, deployed)) != 0) goto _OVER;
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
terrno = 0;
if (buffer == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, buffer, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed);
@ -145,54 +180,58 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write file:%s since %s, deloyed:%d", realfile, terrstr(), deployed);
dError("failed to write file:%s since %s, deloyed:%d", realfile, tstrerror(code), deployed);
}
return code;
}
TdFilePtr dmCheckRunning(const char *dataDir) {
char filepath[PATH_MAX] = {0};
int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) {
int32_t code = 0;
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", filepath, terrstr());
return NULL;
*pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
if (*pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", filepath, tstrerror(code));
return code;
}
int32_t retryTimes = 0;
int32_t ret = 0;
do {
ret = taosLockFile(pFile);
ret = taosLockFile(*pFile);
if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
taosMsleep(1000);
retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes);
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, tstrerror(code), retryTimes);
} while (retryTimes < 12);
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
return NULL;
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(pFile);
*pFile = NULL;
return code;
}
terrno = 0;
dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile;
return code;
}
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool toLogFile){
static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool toLogFile) {
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len);
if (result == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN);
@ -203,48 +242,81 @@ static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool
CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, opts.result, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write checkCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result);
if (pFile != NULL) taosCloseFile(&pFile);
if (result != NULL) taosMemoryFree(result);
return code;
}
static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode, bool toLogFile){
static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptCode, bool toLogFile) {
TdFilePtr pFile = NULL;
int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER;
if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
if (taosWriteFile(pFile, encryptCode, len) <= 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write encryptCode file:%s", realfile);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) {
char *content = NULL;
int64_t size = 0;
TdFilePtr pFile = NULL;
@ -253,31 +325,31 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to open dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to fstat dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content = taosMemoryMalloc(size);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to read dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
encryptDebug("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len);
@ -289,8 +361,8 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
opts.unitLen = 16;
CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
if (strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
code = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
encryptError("failed to compare decrypted result");
goto _OVER;
}
@ -298,9 +370,9 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
encryptDebug("succeed to compare checkCode file:%s", file);
code = 0;
_OVER:
if(result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile);
if (result != NULL) taosMemoryFree(result);
if (content != NULL) taosMemoryFree(content);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
@ -320,38 +392,55 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
int32_t nBytes = snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
terrno = 0;
nBytes = snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to create dir:%s since %s", folder, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to create dir:%s since %s", folder, tstrerror(code));
goto _OVER;
}
if(taosCheckExistFile(realCheckFile)){
if(dmCompareEncryptKey(realCheckFile, key, toLogFile) != 0){
if (taosCheckExistFile(realCheckFile)) {
if ((code = dmCompareEncryptKey(realCheckFile, key, toLogFile)) != 0) {
goto _OVER;
}
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
TAOS_CHECK_GOTO(generateEncryptCode(key, machineId, &encryptCode), &lino, _OVER);
if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){
if ((code = dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile)) != 0) {
goto _OVER;
}
if(dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile) != 0){
if ((code = dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile)) != 0) {
goto _OVER;
}
@ -362,10 +451,9 @@ _OVER:
taosMemoryFree(encryptCode);
taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to update encrypt key since %s", terrstr());
encryptError("failed to update encrypt key at line %d since %s", lino, tstrerror(code));
}
return code;
TAOS_RETURN(code);
#else
return 0;
#endif
@ -373,89 +461,102 @@ _OVER:
extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
static int32_t dmReadEncryptCodeFile(char* file, char** output){
static int32_t dmReadEncryptCodeFile(char *file, char **output) {
TdFilePtr pFile = NULL;
int32_t code = -1;
char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content = taosMemoryMalloc(size + 1);
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER;
}
content[size] = '\0';
*output = content;
content = NULL;
dInfo("succeed to read encryptCode file:%s", file);
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
if (pFile != NULL) taosCloseFile(&pFile);
taosMemoryFree(content);
return code;
}
int32_t dmGetEncryptKey(){
int32_t dmGetEncryptKey() {
#ifdef TD_ENTERPRISE
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char *machineId = NULL;
char *encryptKey = NULL;
char *content = NULL;
int32_t code = -1;
char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0};
char *machineId = NULL;
char *encryptKey = NULL;
char *content = NULL;
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
int32_t nBytes = snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP,
DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(encryptFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if(!taosCheckExistFile(encryptFile)){
nBytes = snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(checkFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if (!taosCheckExistFile(encryptFile)) {
code = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dInfo("no exist, checkCode file:%s", encryptFile);
return 0;
}
if(dmReadEncryptCodeFile(encryptFile, &content) != 0){
if ((code = dmReadEncryptCodeFile(encryptFile, &content)) != 0) {
goto _OVER;
}
if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if(checkAndGetCryptKey(content, machineId, &encryptKey) != 0){
if ((code = checkAndGetCryptKey(content, machineId, &encryptKey)) != 0) {
goto _OVER;
}
taosMemoryFreeClear(machineId);
taosMemoryFreeClear(content);
if(encryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr());
if (encryptKey[0] == '\0') {
code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", tstrerror(code));
goto _OVER;
}
if(dmCompareEncryptKey(checkFile, encryptKey, true) != 0){
if ((code = dmCompareEncryptKey(checkFile, encryptKey, true)) != 0) {
goto _OVER;
}
@ -470,8 +571,7 @@ _OVER:
if (encryptKey != NULL) taosMemoryFree(encryptKey);
if (machineId != NULL) taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to get encrypt key since %s", terrstr());
dError("failed to get encrypt key since %s", tstrerror(code));
}
return code;
#else

60
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -54,6 +54,7 @@ int32_t mndInitAcct(SMnode *pMnode) {
void mndCleanupAcct(SMnode *pMnode) {}
static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
int32_t code = 0;
SAcctObj acctObj = {0};
tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN);
acctObj.createdTime = taosGetTimestampMs();
@ -76,7 +77,10 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
};
SSdbRaw *pRaw = mndAcctActionEncode(&acctObj);
if (pRaw == NULL) return -1;
if (pRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw);
@ -84,21 +88,24 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct");
if (pTrans == NULL) {
sdbFreeRaw(pRaw);
mError("acct:%s, failed to create since %s", acctObj.acct, terrstr());
return -1;
code = terrno;
mError("acct:%s, failed to create since %s", acctObj.acct, tstrerror(code));
TAOS_RETURN(code);
}
mInfo("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr());
code = mndTransAppendCommitlog(pTrans, pRaw);
if (code != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, tstrerror(code));
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
code = mndTransPrepare(pMnode, pTrans);
if (code != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code));
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
mndTransDrop(pTrans);
@ -216,31 +223,36 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
}
static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) {
return -1;
int32_t code = 0;
code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT);
if (code != 0) {
TAOS_RETURN(code);
}
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr());
return -1;
code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", tstrerror(code));
TAOS_RETURN(code);
}
static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) {
return -1;
int32_t code = 0;
code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT);
if (code != 0) {
TAOS_RETURN(code);
}
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr());
return -1;
code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", tstrerror(code));
TAOS_RETURN(code);
}
static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) {
return -1;
int32_t code = 0;
if ((code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT)) != 0) {
TAOS_RETURN(code);
}
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr());
return -1;
code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", tstrerror(code));
TAOS_RETURN(code);
}

197
source/dnode/mnode/impl/src/mndArbGroup.c
View File

@ -57,6 +57,7 @@ static int32_t mndArbCheckToken(const char *token1, const char *token2) {
}
int32_t mndInitArbGroup(SMnode *pMnode) {
int32_t code = 0;
SSdbTable table = {
.sdbType = SDB_ARBGROUP,
.keyType = SDB_KEY_INT32,
@ -78,6 +79,10 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup);
arbUpdateHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
if (arbUpdateHash == NULL) {
code = terrno;
TAOS_RETURN(code);
}
return sdbSetTable(pMnode->pSdb, table);
}
@ -248,50 +253,74 @@ _OVER:
}
int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
if (pRedoRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendRedolog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING)) != 0) TAOS_RETURN(code);
return 0;
}
int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndArbGroupActionEncode(pGroup);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
if (pUndoRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendUndolog(pTrans, pUndoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED)) != 0) TAOS_RETURN(code);
return 0;
}
int32_t mndSetCreateArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if (pCommitRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw) != 0)) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) TAOS_RETURN(code);
return 0;
}
int32_t mndSetDropArbGroupPrepareLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
if (pRedoRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)) != 0) TAOS_RETURN(code);
return 0;
}
static int32_t mndSetDropArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
if (pRedoRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendRedolog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)) != 0) TAOS_RETURN(code);
return 0;
}
int32_t mndSetDropArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
if (pCommitRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) TAOS_RETURN(code);
return 0;
}
@ -343,6 +372,7 @@ static int32_t mndSendArbHeartBeatReq(SDnodeObj *pDnode, char *arbToken, int64_t
}
static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL;
@ -377,7 +407,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
}
char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) {
if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-hb timer");
pIter = taosHashIterate(pDnodeHash, NULL);
while (pIter) {
@ -386,7 +416,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
pIter = taosHashIterate(pDnodeHash, pIter);
}
taosHashCleanup(pDnodeHash);
return -1;
TAOS_RETURN(code);
}
int64_t nowMs = taosGetTimestampMs();
@ -447,6 +477,7 @@ static void *mndBuildArbCheckSyncReq(int32_t *pContLen, int32_t vgId, char *arbT
static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbToken, int64_t term, char *member0Token,
char *member1Token) {
int32_t code = 0;
int32_t contLen = 0;
void *pHead = mndBuildArbCheckSyncReq(&contLen, vgId, arbToken, term, member0Token, member1Token);
if (!pHead) {
@ -459,10 +490,12 @@ static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbTok
if (epSet.numOfEps == 0) {
mError("vgId:%d, failed to send check-sync request since no epSet found", vgId);
rpcFreeCont(pHead);
return -1;
code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, failed to send check-sync request since 0x%x", vgId, code);
} else {
@ -502,11 +535,14 @@ static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, ch
static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken,
int64_t term, char *memberToken) {
int32_t code = 0;
int32_t contLen = 0;
void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken);
if (!pHead) {
mError("vgId:%d, failed to build set-assigned request", vgId);
return -1;
code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
SRpcMsg rpcMsg = {.msgType = TDMT_SYNC_SET_ASSIGNED_LEADER, .pCont = pHead, .contLen = contLen};
@ -514,9 +550,11 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
if (epSet.numOfEps == 0) {
mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since no epSet found", dnodeId, vgId);
rpcFreeCont(pHead);
return -1;
code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since 0x%x", dnodeId, vgId, code);
} else {
@ -526,6 +564,7 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
}
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL;
@ -533,14 +572,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
void *pIter = NULL;
char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) {
if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-check-sync timer");
return -1;
TAOS_RETURN(code);
}
int64_t term = mndGetTerm(pMnode);
if (term < 0) {
mError("arb failed to get term since %s", terrstr());
return -1;
code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
int64_t roleTimeMs = mndGetRoleTimeMs(pMnode);
@ -747,13 +788,13 @@ _OVER:
}
static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
int ret = -1;
int code = -1;
size_t sz = 0;
SMArbUpdateGroupBatchReq req = {0};
if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) {
if ((code = tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req)) != 0) {
mError("arb failed to decode arb-update-group request");
return -1;
TAOS_RETURN(code);
}
SMnode *pMnode = pReq->info.node;
@ -789,7 +830,7 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
mndTransAddArbGroupId(pTrans, newGroup.vgId);
if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) {
if ((code = mndSetCreateArbGroupCommitLogs(pTrans, &newGroup)) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id,
terrstr());
goto _OVER;
@ -803,13 +844,13 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pOldGroup);
}
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) goto _OVER;
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
ret = 0;
code = 0;
_OVER:
if (ret != 0) {
if (code != 0) {
// failed to update arbgroup
for (size_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
@ -819,7 +860,7 @@ _OVER:
mndTransDrop(pTrans);
tFreeSMArbUpdateGroupBatchReq(&req);
return ret;
return code;
}
static void mndArbGroupDupObj(SArbGroup *pGroup, SArbGroup *pNew) {
@ -841,10 +882,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) {
}
static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
int32_t ret = -1;
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup");
if (pTrans == NULL) {
mError("failed to update arbgroup in create trans, vgId:%d, since %s", pNew->vgId, terrstr());
if (terrno != 0) code = terrno;
goto _OVER;
}
@ -854,23 +896,22 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
pNew->assignedLeader.token, pNew->assignedLeader.acked);
mndTransAddArbGroupId(pTrans, pNew->vgId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
ret = -1;
if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) {
goto _OVER;
}
if (mndSetCreateArbGroupCommitLogs(pTrans, pNew) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, since %s", pNew->vgId, terrstr());
if ((code = mndSetCreateArbGroupCommitLogs(pTrans, pNew)) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, since %s", pNew->vgId, tstrerror(code));
goto _OVER;
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
ret = 0;
code = 0;
_OVER:
mndTransDrop(pTrans);
return ret;
return code;
}
bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMember, int64_t nowMs, int32_t dnodeId,
@ -993,11 +1034,13 @@ _OVER:
}
static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token, char *member1Token, bool newIsSync) {
int32_t code = 0;
SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
if (pGroup == NULL) {
terrno = TSDB_CODE_NOT_FOUND;
mInfo("failed to update arb sync, vgId:%d not found", vgId);
return -1;
code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
SArbGroup newGroup = {0};
@ -1018,38 +1061,36 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
return 0;
}
int32_t ret = -1;
int32_t code = -1;
SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) {
if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-hb response");
terrno = TSDB_CODE_NOT_FOUND;
return -1;
TAOS_RETURN(code);
}
SVArbHeartBeatRsp arbHbRsp = {0};
if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) {
if ((code = tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp)) != 0) {
mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
TAOS_RETURN(code);
}
if (mndArbCheckToken(arbToken, arbHbRsp.arbToken) != 0) {
mInfo("arb hearbeat skip update for dnodeId:%d, arb token mismatch, local:[%s] msg:[%s]", arbHbRsp.dnodeId,
arbToken, arbHbRsp.arbToken);
terrno = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
code = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
goto _OVER;
}
(void)mndUpdateArbHeartBeat(pMnode, arbHbRsp.dnodeId, arbHbRsp.hbMembers);
ret = 0;
code = 0;
_OVER:
tFreeSVArbHeartBeatRsp(&arbHbRsp);
return ret;
return code;
}
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
@ -1058,27 +1099,25 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
return 0;
}
int32_t ret = -1;
int32_t code = -1;
SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) {
if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-check-sync response");
terrno = TSDB_CODE_NOT_FOUND;
return -1;
TAOS_RETURN(code);
}
SVArbCheckSyncRsp syncRsp = {0};
if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) {
if ((code = tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp)) != 0) {
mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code));
if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
terrno = TSDB_CODE_SUCCESS;
return 0;
}
terrno = TSDB_CODE_INVALID_MSG;
return -1;
TAOS_RETURN(code);
}
if (mndArbCheckToken(arbToken, syncRsp.arbToken) != 0) {
@ -1089,16 +1128,16 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
}
bool newIsSync = (syncRsp.errCode == TSDB_CODE_SUCCESS);
if (mndUpdateArbSync(pMnode, syncRsp.vgId, syncRsp.member0Token, syncRsp.member1Token, newIsSync) != 0) {
if ((code = mndUpdateArbSync(pMnode, syncRsp.vgId, syncRsp.member0Token, syncRsp.member1Token, newIsSync)) != 0) {
mInfo("failed to update arb sync for vgId:%d, since:%s", syncRsp.vgId, terrstr());
goto _OVER;
}
ret = 0;
code = 0;
_OVER:
tFreeSVArbCheckSyncRsp(&syncRsp);
return ret;
TAOS_RETURN(code);
}
bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char *memberToken, int32_t errcode,
@ -1138,35 +1177,35 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
return 0;
}
int32_t ret = -1;
int32_t code = -1;
SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) {
if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-set-assigned response");
terrno = TSDB_CODE_NOT_FOUND;
return -1;
TAOS_RETURN(code);
}
SVArbSetAssignedLeaderRsp setAssignedRsp = {0};
if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) {
if ((code = tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp)) != 0) {
mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
TAOS_RETURN(code);
}
if (mndArbCheckToken(arbToken, setAssignedRsp.arbToken) != 0) {
mInfo("skip update arb assigned for vgId:%d, arb token mismatch, local:[%s] msg:[%s]", setAssignedRsp.vgId,
arbToken, setAssignedRsp.arbToken);
terrno = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
code = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
goto _OVER;
}
SArbGroup *pGroup = mndAcquireArbGroup(pMnode, setAssignedRsp.vgId);
if (!pGroup) {
mError("failed to set arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr());
code = -1;
if (terrno != 0) code = terrno;
goto _OVER;
}
@ -1174,17 +1213,17 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(pGroup, setAssignedRsp.vgId, setAssignedRsp.memberToken,
pRsp->code, &newGroup);
if (updateAssigned) {
if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) {
mInfo("failed to pullup update arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr());
if ((code = mndPullupArbUpdateGroup(pMnode, &newGroup)) != 0) {
mInfo("failed to pullup update arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, tstrerror(code));
goto _OVER;
}
}
ret = 0;
code = 0;
_OVER:
tFreeSVArbSetAssignedLeaderRsp(&setAssignedRsp);
return ret;
return code;
}
static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

58
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -14,8 +14,8 @@
*/
#define _DEFAULT_SOURCE
#include "audit.h"
#include "mndCluster.h"
#include "audit.h"
#include "mndGrant.h"
#include "mndPrivilege.h"
#include "mndShow.h"
@ -61,11 +61,13 @@ int32_t mndInitCluster(SMnode *pMnode) {
void mndCleanupCluster(SMnode *pMnode) {}
int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
SSdb *pSdb = pMnode->pSdb;
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId);
if (pCluster == NULL) {
return -1;
code = terrno;
TAOS_RETURN(code);
}
tstrncpy(clusterName, pCluster->name, len);
@ -247,7 +249,10 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
mInfo("cluster:%" PRId64 ", name is %s", clusterObj.id, clusterObj.name);
SSdbRaw *pRaw = mndClusterActionEncode(&clusterObj);
if (pRaw == NULL) return -1;
if (pRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw);
@ -256,21 +261,22 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
if (pTrans == NULL) {
sdbFreeRaw(pRaw);
mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr());
return -1;
code = terrno;
TAOS_RETURN(code);
}
mInfo("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
mndTransDrop(pTrans);
@ -354,22 +360,33 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
return 0;
}
int32_t code = 0;
mInfo("update cluster uptime to %d", clusterObj.upTime);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime");
if (pTrans == NULL) return -1;
if (pTrans == NULL) {
code = terrno;
TAOS_RETURN(code);
}
SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
if (pCommitRaw == NULL) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
mndTransDrop(pTrans);
@ -381,13 +398,12 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SMCfgClusterReq cfgReq = {0};
if (tDeserializeSMCfgClusterReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
TAOS_RETURN(code);
}
mInfo("cluster: start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER) != 0) {
code = terrno != 0 ? terrno : TSDB_CODE_MND_NO_RIGHTS;
if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER)) != 0) {
goto _exit;
}
@ -417,17 +433,17 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
}
{ // audit
auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1));
auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql,
TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1));
}
_exit:
tFreeSMCfgClusterReq(&cfgReq);
if (code != 0) {
terrno = code;
mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr());
} else {
mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value);
}
return code;
TAOS_RETURN(code);
}
int32_t mndProcessConfigClusterRsp(SRpcMsg *pRsp) {

183
source/dnode/mnode/impl/src/mndCompact.c
View File

@ -75,11 +75,11 @@ int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
TAOS_CHECK_RETURN(tStartDecode(&decoder));
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeCStrTo(&decoder, pObj->dbname) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactId));
TAOS_CHECK_RETURN(tDecodeCStrTo(&decoder, pObj->dbname));
TAOS_CHECK_RETURN(tDecodeI64(&decoder, &pObj->startTime));
tEndDecode(&decoder);
@ -174,13 +174,10 @@ SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
if ((terrno = tDeserializeSCompactObj(buf, tlen, pCompact)) < 0) {
goto OVER;
}
// taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
@ -227,6 +224,7 @@ void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
// compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) {
int32_t code = 0;
pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name);
@ -234,10 +232,14 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompac
pCompact->startTime = taosGetTimestampMs();
SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
if (pVgRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pVgRaw) != 0) {
if (pVgRaw == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendPrepareLog(pTrans, pVgRaw)) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
@ -335,44 +337,62 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId,
int32_t dnodeid) {
int32_t code = 0;
STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
if (pDnode == NULL) return -1;
if (pDnode == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0;
void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid);
if (pReq == NULL) return -1;
if (pReq == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_KILL_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
}
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
mndTransSetDbName(pTrans, pCompact->dbname, NULL);
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
@ -386,14 +406,20 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if (pVgroup == NULL) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if (mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0) {
if ((code = mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId)) != 0) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
mndReleaseVgroup(pMnode, pVgroup);
@ -412,10 +438,10 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
sdbRelease(pMnode->pSdb, pDetail);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
mndTransDrop(pTrans);
@ -423,30 +449,27 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
}
int32_t mndProcessKillCompactReq(SRpcMsg *pReq) {
int32_t code = 0;
int32_t lino = 0;
SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
if ((code = tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq)) != 0) {
TAOS_RETURN(code);
}
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if (pCompact == NULL) {
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID;
code = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq);
return -1;
TAOS_RETURN(code);
}
if (0 != mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB)) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB), &lino, _OVER);
if (mndKillCompact(pMnode, pReq, pCompact) < 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndKillCompact(pMnode, pReq, pCompact), &lino, _OVER);
code = TSDB_CODE_ACTION_IN_PROGRESS;
@ -463,12 +486,14 @@ _OVER:
tFreeSKillCompactReq(&killCompactReq);
sdbRelease(pMnode->pSdb, pCompact);
return code;
TAOS_RETURN(code);
}
// update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId,
SQueryCompactProgressRsp *rsp) {
int32_t code = 0;
void *pIter = NULL;
while (1) {
SCompactDetailObj *pDetail = NULL;
@ -479,9 +504,10 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
pDetail->newNumberFileset = rsp->numberFileset;
pDetail->newFinished = rsp->finished;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
return 0;
TAOS_RETURN(code);
}
sdbRelease(pMnode->pSdb, pDetail);
@ -491,14 +517,13 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
}
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
SQueryCompactProgressRsp req = {0};
int32_t code = 0;
SQueryCompactProgressRsp req = {0};
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
if (code != 0) {
terrno = TSDB_CODE_INVALID_MSG;
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont,
pReq->contLen);
return -1;
TAOS_RETURN(code);
}
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
@ -508,13 +533,12 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
if (code != 0) {
terrno = code;
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
// timer
@ -531,7 +555,10 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if (pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port);
if (addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port) != 0) {
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
mndReleaseDnode(pMnode, pDnode);
SQueryCompactProgressReq req;
@ -541,8 +568,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
@ -551,7 +576,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
@ -563,13 +587,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen};
// rpcMsg.pCont = rpcMallocCont(contLen);
// if (rpcMsg.pCont == NULL) {
// return;
// }
// memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead;
char detail[1024] = {0};
@ -589,6 +606,7 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
}
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
int32_t code = 0;
bool needSave = false;
void *pIter = NULL;
while (1) {
@ -612,7 +630,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
}
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
if (pCompact == NULL) return 0;
if (pCompact == NULL) TAOS_RETURN(code);
SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) {
@ -625,13 +643,15 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (!needSave) {
mDebug("compact:%" PRId32 ", no need to save", compactId);
return 0;
TAOS_RETURN(code);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
@ -654,10 +674,20 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
if (pCommitRaw == NULL) {
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
}
@ -678,11 +708,13 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pDetail->numberFileset == -1 && pDetail->finished == -1) {
allFinished = false;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
break;
}
if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) {
allFinished = false;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
break;
}
@ -710,11 +742,19 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
if (pCommitRaw == NULL) {
mndTransDrop(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id,
terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId);
@ -724,20 +764,26 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
}
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
if (pCommitRaw == NULL) {
mndTransDrop(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compact action", pCompact->compactId);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans);
sdbRelease(pMnode->pSdb, pCompact);
return -1;
TAOS_RETURN(code);
}
sdbRelease(pMnode->pSdb, pCompact);
@ -746,6 +792,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
}
void mndCompactPullup(SMnode *pMnode) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t));
if (pArray == NULL) return;
@ -766,7 +813,9 @@ void mndCompactPullup(SMnode *pMnode) {
if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact);
mndSaveCompactProgress(pMnode, pCompact->compactId);
if ((code = mndSaveCompactProgress(pMnode, pCompact->compactId)) != 0) {
mError("compact:%d, failed to save compact progress since %s", pCompact->compactId, tstrerror(code));
}
}
mndReleaseCompact(pMnode, pCompact);
}

34
source/dnode/mnode/impl/src/mndCompactDetail.c
View File

@ -125,17 +125,17 @@ int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailO
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactDetailId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->vgId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->dnodeId) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->numberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->finished) < 0) return -1;
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newNumberFileset) < 0) return -1;
if (tDecodeI32(&decoder, &pObj->newFinished) < 0) return -1;
TAOS_CHECK_RETURN(tStartDecode(&decoder));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactDetailId));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactId));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->vgId));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->dnodeId));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->numberFileset));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->finished));
TAOS_CHECK_RETURN(tDecodeI64(&decoder, &pObj->startTime));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->newNumberFileset));
TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->newFinished));
tEndDecode(&decoder);
@ -231,13 +231,10 @@ SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) {
}
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactDetailObj(buf, tlen, pCompact) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
if ((terrno = tDeserializeSCompactDetailObj(buf, tlen, pCompact)) < 0) {
goto OVER;
}
//taosInitRWLatch(&pView->lock);
OVER:
taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) {
@ -274,6 +271,7 @@ int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact,
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index){
int32_t code = 0;
SCompactDetailObj compactDetail = {0};
compactDetail.compactDetailId = index;
compactDetail.compactId = pCompact->compactId;
@ -292,9 +290,11 @@ int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* p
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0;
TAOS_RETURN(code);
}

129
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -95,26 +95,21 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
bool enableReplay) {
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
int32_t lino = 0;
int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) {
char *pOneTopic = taosArrayGetP(pTopicList, i);
pTopic = mndAcquireTopic(pMnode, pOneTopic);
if (pTopic == NULL) { // terrno has been set by callee function
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED;
}
if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) {
code = TSDB_CODE_MND_NO_RIGHTS;
terrno = TSDB_CODE_MND_NO_RIGHTS;
goto FAILED;
}
TAOS_CHECK_GOTO(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic), &lino, FAILED);
if ((terrno = grantCheckExpire(TSDB_GRANT_SUBSCRIPTION)) < 0) {
code = terrno;
goto FAILED;
}
TAOS_CHECK_GOTO(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION), &lino, FAILED);
if (enableReplay) {
if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
@ -123,7 +118,8 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
} else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
if (pDb == NULL) {
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED;
}
if (pDb->cfg.numOfVgroups != 1) {
@ -141,11 +137,12 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
return 0;
FAILED:
mndReleaseTopic(pMnode, pTopic);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL;
@ -153,7 +150,8 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
if (pConsumer == NULL) {
mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId);
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
@ -161,42 +159,39 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
pConsumer->status, mndConsumerStatusName(pConsumer->status));
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
code = -1;
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto END;
}
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL);
if (pConsumerNew == NULL){
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) {
code = -1;
goto END;
}
code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
if (code != 0) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
code = mndSetConsumerCommitLogs(pTrans, pConsumerNew);
if (code != 0) {
goto END;
}
TAOS_CHECK_GOTO(validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false), &lino, END);
TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), &lino, END);
code = mndTransPrepare(pMnode, pTrans);
END:
mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node;
SMqConsumerClearMsg *pClearMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL;
@ -205,7 +200,7 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId);
return 0;
TAOS_RETURN(code);
}
mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId,
@ -213,21 +208,20 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL);
if (pConsumerNew == NULL){
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm");
if (pTrans == NULL) {
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
// this is the drop action, not the update action
code = mndSetConsumerDropLogs(pTrans, pConsumerNew);
if (code != 0) {
goto END;
}
TAOS_CHECK_GOTO(mndSetConsumerDropLogs(pTrans, pConsumerNew), &lino, END);
code = mndTransPrepare(pMnode, pTrans);
@ -235,7 +229,7 @@ END:
mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) {
@ -310,10 +304,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
SMqHbRsp rsp = {0};
SMqConsumerObj *pConsumer = NULL;
if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
}
TAOS_CHECK_GOTO(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req), NULL, end);
int64_t consumerId = req.consumerId;
pConsumer = mndAcquireConsumer(pMnode, consumerId);
@ -322,10 +313,8 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto end;
}
code = checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user);
if (code != 0) {
goto end;
}
TAOS_CHECK_GOTO(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user), NULL, end);
atomic_store_32(&pConsumer->hbStatus, 0);
@ -343,7 +332,7 @@ end:
tDestroySMqHbRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer);
tDestroySMqHbReq(&req);
return code;
TAOS_RETURN(code);
}
static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){
@ -431,6 +420,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
}
static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoch, int64_t consumerId){
int32_t code = 0;
// encode rsp
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp);
void *buf = rpcMallocCont(tlen);
@ -452,7 +442,7 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc
// send rsp
pMsg->info.rsp = buf;
pMsg->info.rspLen = tlen;
return 0;
TAOS_RETURN(code);
}
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
@ -461,9 +451,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
SMqAskEpRsp rsp = {0};
int32_t code = 0;
if (tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TAOS_CHECK_RETURN(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req));
int64_t consumerId = req.consumerId;
@ -517,19 +505,29 @@ END:
}
int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static void freeItem(void *param) {
@ -585,6 +583,7 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb
}
static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
int32_t code = 0;
taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
@ -592,11 +591,10 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
for (int i = 0; i < newTopicNum; i++) {
int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i));
if (gNum >= MND_MAX_GROUP_PER_TOPIC) {
terrno = TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
return -1;
return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
}
}
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){
@ -672,7 +670,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over;
}
@ -683,21 +682,21 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pConsumerNew = buildSubConsumer(pMnode, &subscribe);
if(pConsumerNew == NULL){
code = -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over;
}
code = mndSetConsumerCommitLogs(pTrans, pConsumerNew);
if (code != 0) goto _over;
code = mndTransPrepare(pMnode, pTrans);
if (code != 0) goto _over;
TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), NULL, _over);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _over);
code = TSDB_CODE_ACTION_IN_PROGRESS;
_over:
mndTransDrop(pTrans);
tDeleteSMqConsumerObj(pConsumerNew);
taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
return code;
TAOS_RETURN(code);
}
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {

687
source/dnode/mnode/impl/src/mndDb.c
View File

File diff suppressed because it is too large Load Diff

31
source/dnode/mnode/impl/src/mndMain.c
View File

@ -464,26 +464,23 @@ static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
static int32_t mndInitWal(SMnode *pMnode) {
char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = {
.vgId = 1,
.fsyncPeriod = 0,
.rollPeriod = -1,
.segSize = -1,
.retentionPeriod = 0,
.retentionSize = 0,
.level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}
};
SWalCfg cfg = {.vgId = 1,
.fsyncPeriod = 0,
.rollPeriod = -1,
.segSize = -1,
.retentionPeriod = 0,
.retentionSize = 0,
.level = TAOS_WAL_FSYNC,
.encryptAlgorithm = 0,
.encryptKey = {0}};
#if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){
if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL) {
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL) ? tsiEncryptAlgorithm : 0;
if (tsEncryptKey[0] == '\0') {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
}
else{
} else {
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
@ -856,7 +853,7 @@ _OVER:
return -1;
}
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
SMnode *pMnode = pMsg->info.node;
const STraceId *trace = &pMsg->info.traceId;
int32_t code = TSDB_CODE_SUCCESS;

256
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -69,6 +69,7 @@ int32_t mndInitMnode(SMnode *pMnode) {
void mndCleanupMnode(SMnode *pMnode) {}
SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
terrno = 0;
SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
@ -82,13 +83,18 @@ void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
}
static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
int32_t code = 0;
SMnodeObj mnodeObj = {0};
mnodeObj.id = 1;
mnodeObj.createdTime = taosGetTimestampMs();
mnodeObj.updateTime = mnodeObj.createdTime;
SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj);
if (pRaw == NULL) return -1;
if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw);
@ -97,25 +103,27 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
if (pTrans == NULL) {
sdbFreeRaw(pRaw);
mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
mInfo("trans:%d, used to create mnode:%d", pTrans->id, mnodeObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) {
if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
return 0;
TAOS_RETURN(code);
}
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) {
@ -188,16 +196,19 @@ _OVER:
}
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
int32_t code = 0;
mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) {
mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
int32_t code = 0;
}
pObj->syncState = TAOS_SYNC_STATE_OFFLINE;
mndReloadSyncConfig(pSdb->pMnode);
return 0;
TAOS_RETURN(code);
}
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
@ -271,38 +282,59 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
}
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
int32_t mndSetRestoreCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
@ -315,15 +347,16 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p
.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq,
SEpSet *pAlterMnodeTypeEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq);
@ -337,14 +370,15 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeType
.acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
@ -357,15 +391,16 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA
.acceptableCode = 0,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq);
void *pReq = taosMemoryMalloc(contLen);
tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq);
@ -378,11 +413,11 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop
.acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED,
};
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -426,9 +461,9 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -474,9 +509,9 @@ int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -517,9 +552,9 @@ static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -565,19 +600,23 @@ int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0;
TAOS_RETURN(0);
}
static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
SMnodeObj mnodeObj = {0};
mnodeObj.id = pDnode->id;
@ -586,8 +625,8 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeObj.role = TAOS_SYNC_ROLE_LEARNER;
mnodeObj.lastIndex = pMnode->applied;
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj), NULL, _OVER);
SMnodeObj mnodeLeaderObj = {0};
mnodeLeaderObj.id = pDnode->id;
@ -596,15 +635,15 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER;
mnodeLeaderObj.lastIndex = pMnode->applied + 1;
if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER;
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
@ -614,19 +653,14 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
SMCreateMnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("mnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE), NULL, _OVER);
pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
code = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
goto _OVER;
} else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) {
goto _OVER;
@ -634,17 +668,17 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER;
}
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
terrno = TSDB_CODE_MND_TOO_MANY_MNODES;
code = TSDB_CODE_MND_TOO_MANY_MNODES;
goto _OVER;
}
if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE;
code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER;
}
@ -665,27 +699,38 @@ _OVER:
mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj,
bool force) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SDDropMnodeReq dropReq = {0};
@ -700,32 +745,32 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
if (totalMnodes == 2) {
if (force) {
mError("cant't force drop dnode, since a mnode on it and replica is 2");
terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
return -1;
code = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
TAOS_RETURN(code);
}
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
}
} else if (totalMnodes == 3) {
mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1;
TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
}
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
} else {
return -1;
TAOS_RETURN(-1);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
pObj->lastIndex = pMnode->applied;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1;
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force));
TAOS_CHECK_RETURN(mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj));
return 0;
}
@ -734,19 +779,23 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) {
STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
@ -755,38 +804,35 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
SMnodeObj *pObj = NULL;
SMDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("mnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER;
code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER;
}
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES;
code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER;
}
if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE;
code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER;
}
@ -805,7 +851,7 @@ _OVER:
mndReleaseMnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
@ -892,13 +938,11 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
#if 1
return 0;
#else
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0};
if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq));
SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
@ -913,9 +957,9 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return 0;
}
if (mndWriteFile(pMnode->path, &option) != 0) {
if ((code = mndWriteFile(pMnode->path, &option)) != 0) {
mError("failed to write mnode file since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
@ -939,14 +983,14 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
}
}
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) {
mError("failed to sync reconfig since %s", terrstr());
} else {
mInfo("alter mnode sync success");
}
return code;
TAOS_RETURN(code);
#endif
}

53
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -19,10 +19,11 @@
// connection/application/
int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < colNum; ++i) {
@ -34,10 +35,11 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S
}
*pDst = schema;
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
int32_t mndPerfsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -53,56 +55,56 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName));
meta.numOfColumns = pSysDbTableMeta[i].colNum;
if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) {
return -1;
}
TAOS_CHECK_RETURN(mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas));
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
}
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
*pRsp = *meta;
pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema));
return 0;
TAOS_RETURN(code);
}
int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == pMeta) {
mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
strcpy(pRsp->tbName, pMeta->tbName);
@ -114,20 +116,21 @@ int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *t
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0;
TAOS_RETURN(code);
}
int32_t mndInitPerfs(SMnode *pMnode) {
int32_t code = 0;
pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pMnode->perfsMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
return mndPerfsInitMeta(pMnode->perfsMeta);

77
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -85,22 +85,23 @@ static void mndCancelGetNextApp(SMnode *pMnode, void *pIter);
static int32_t mndProcessSvrVerReq(SRpcMsg *pReq);
int32_t mndInitProfile(SMnode *pMnode) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
// in ms
int32_t checkTime = tsShellActivityTimer * 2 * 1000;
pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->connCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app");
if (pMgmt->appCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr());
return -1;
TAOS_RETURN(code);
}
mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq);
@ -116,7 +117,7 @@ int32_t mndInitProfile(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp);
return 0;
TAOS_RETURN(code);
}
void mndCleanupProfile(SMnode *pMnode) {
@ -384,6 +385,7 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq)
static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); }
static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) {
terrno = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId));
@ -431,13 +433,16 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) {
}
static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) {
int32_t code = 0;
SAppHbReq *pReq = &pHbReq->app;
SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId);
if (pApp == NULL) {
pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq);
if (pApp == NULL) {
mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else {
mDebug("a new app %" PRIx64 " is created", pReq->appId);
mndReleaseApp(pMnode, pApp);
@ -475,6 +480,7 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) {
static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq,
SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
SRpcConnInfo connInfo = pMsg->info.conn;
@ -492,7 +498,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
pHbReq->app.pid, pHbReq->app.name, 0);
if (pConn == NULL) {
mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr());
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else {
mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id);
}
@ -501,9 +509,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic));
if (rspBasic == NULL) {
mndReleaseConn(pMnode, pConn, true);
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr());
return -1;
TAOS_RETURN(code);
}
mndSaveQueryList(pConn, pBasic);
@ -539,9 +547,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
hbRsp.info = taosArrayInit(kvNum, sizeof(SKv));
if (NULL == hbRsp.info) {
mError("taosArrayInit %d rsp kv failed", kvNum);
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
tFreeClientHbRsp(&hbRsp);
return -1;
TAOS_RETURN(code);
}
#ifdef TD_ENTERPRISE
@ -554,8 +562,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer);
SDynViewVersion *pRspVer = NULL;
if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) {
return -1;
if (0 != (code = mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer))) {
TAOS_RETURN(code);
}
if (needCheck) {
@ -647,13 +655,14 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
}
static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SClientHbBatchReq batchReq = {0};
if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) {
taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = TSDB_CODE_INVALID_MSG;
TAOS_RETURN(code);
}
SConnPreparedObj obj = {0};
@ -699,31 +708,27 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
taosArrayDestroy(obj.pQnodeList);
return 0;
TAOS_RETURN(code);
}
static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillQueryReq killReq = {0};
if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq));
mInfo("kill query msg is received, queryId:%s", killReq.queryStrId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY));
int32_t connId = 0;
uint64_t queryId = 0;
char *p = strchr(killReq.queryStrId, ':');
if (NULL == p) {
mError("invalid query id %s", killReq.queryStrId);
terrno = TSDB_CODE_MND_INVALID_QUERY_ID;
return -1;
code = TSDB_CODE_MND_INVALID_QUERY_ID;
TAOS_RETURN(code);
}
*p = 0;
connId = taosStr2Int32(killReq.queryStrId, NULL, 16);
@ -732,40 +737,36 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t));
if (pConn == NULL) {
mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID;
return -1;
code = TSDB_CODE_MND_INVALID_CONN_ID;
TAOS_RETURN(code);
} else {
mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user);
pConn->killId = queryId;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return 0;
TAOS_RETURN(code);
}
}
static int32_t mndProcessKillConnReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillConnReq killReq = {0};
if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq));
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN));
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t));
if (pConn == NULL) {
mError("connId:%u, failed to kill connection, conn not exist", killReq.connId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID;
return -1;
code = TSDB_CODE_MND_INVALID_CONN_ID;
TAOS_RETURN(code);
} else {
mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user);
pConn->killed = 1;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return TSDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
}

190
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -138,15 +138,16 @@ _OVER:
}
static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) {
int32_t code = 0;
mTrace("qnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) {
@ -166,27 +167,42 @@ static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew
}
static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
@ -194,6 +210,7 @@ bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
}
int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0};
createReq.dnodeId = pDnode->id;
@ -212,23 +229,24 @@ int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeOb
action.msgType = TDMT_DND_CREATE_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +257,12 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) {
@ -256,22 +274,26 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
qnodeObj.updateTime = qnodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER;
if (mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
@ -281,19 +303,14 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
SMCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("qnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE), NULL, _OVER);
pObj = mndAcquireQnode(pMnode, createReq.dnodeId);
if (pObj != NULL) {
terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST;
code = TSDB_CODE_MND_QNODE_ALREADY_EXIST;
goto _OVER;
} else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) {
goto _OVER;
@ -301,7 +318,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER;
}
@ -320,34 +337,47 @@ _OVER:
mndReleaseQnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if (mndTransAppendCommitlog(pTrans, pCommitRaw))
;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED))
;
TAOS_RETURN(code);
}
static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -358,20 +388,20 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1;
if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropQnodeRedoLogs(pTrans, pObj));
TAOS_CHECK_RETURN(mndSetDropQnodeCommitLogs(pTrans, pObj));
if (!force) {
if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj));
}
return 0;
}
@ -380,18 +410,22 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
@ -400,23 +434,20 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
SQnodeObj *pObj = NULL;
SMDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("qnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE), NULL, _OVER);
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireQnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
@ -435,10 +466,11 @@ _OVER:
mndReleaseQnode(pMnode, pObj);
tFreeSDDropQnodeReq(&dropReq);
return code;
TAOS_RETURN(code);
}
int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SQnodeObj *pObj = NULL;
@ -447,8 +479,8 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
if (NULL == qnodeList) {
mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
while (1) {
@ -484,20 +516,14 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
SQnodeListReq qlistReq = {0};
SQnodeListRsp qlistRsp = {0};
if (tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq) != 0) {
mError("failed to parse qnode list req");
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq), NULL, _OVER);
if (mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum), NULL, _OVER);
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -509,7 +535,7 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
_OVER:
tFreeSQnodeListRsp(&qlistRsp);
return code;
TAOS_RETURN(code);
}
static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

6
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -88,7 +88,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) {
if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
mError("tDeserializeSBatchReq failed");
goto _exit;
@ -119,7 +119,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req->msgType)];
if (fp == NULL) {
mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
code = TSDB_CODE_MSG_NOT_PROCESSED;
taosArrayDestroy(batchRsp.pRsps);
return -1;
}
@ -164,7 +164,7 @@ _exit:
taosArrayDestroyEx(batchReq.pMsgs, tFreeSBatchReqMsg);
taosArrayDestroyEx(batchRsp.pRsps, mnodeFreeSBatchRspMsg);
return code;
TAOS_RETURN(code);
}
int32_t mndInitQuery(SMnode *pMnode) {

145
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -50,17 +50,17 @@ static bool isCountWindowStreamTask(SSubplan* pPlan) {
int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
int64_t watermark, int64_t deleteMark) {
int32_t code = 0;
SNode* pAst = NULL;
SQueryPlan* pPlan = NULL;
terrno = TSDB_CODE_SUCCESS;
if (nodesStringToNode(ast, &pAst) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
if (qSetSTableIdForRsma(pAst, uid) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
@ -75,33 +75,33 @@ int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64
};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
int32_t opNum = LIST_LENGTH(inner->pNodeList);
if (opNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
if (qSubPlanToString(plan, pDst, pDstLen) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
code = TSDB_CODE_QRY_INVALID_INPUT;
goto END;
}
END:
if (pAst) nodesDestroyNode(pAst);
if (pPlan) nodesDestroyNode((SNode*)pPlan);
return terrno;
TAOS_RETURN(code);
}
int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
@ -127,6 +127,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList,
SStreamTask* pTask) {
int32_t code = 0;
bool isShuffle = false;
if (pStream->fixedSinkVgId == 0) {
@ -135,9 +136,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
isShuffle = true;
pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH;
pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH;
if (mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL) < 0) {
return -1;
}
TAOS_CHECK_RETURN(mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL));
}
sdbRelease(pMnode->pSdb, pDb);
@ -166,7 +165,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
streamTaskSetFixedDownstreamInfo(pTask, pOneSinkTask);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) {
@ -242,11 +241,11 @@ static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgrou
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList,
pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
SStreamTask* pTask = NULL;
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory,
pStream->subTableWithoutMd5, &pTask);
if (code != 0) {
return code;
}
mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
@ -353,18 +352,13 @@ static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFi
}
}
static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) {
static int32_t buildSourceTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam, SStreamTask** pTask) {
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pTask =
tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pTask == NULL) {
return NULL;
}
return pTask;
int32_t code = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pTask);
return code;
}
static void addNewTaskList(SStreamObj* pStream) {
@ -398,10 +392,10 @@ static void setHTasksId(SStreamObj* pStream) {
static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) {
SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
SStreamTask* pTask = NULL;
int32_t code = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
@ -412,10 +406,9 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
if (code != 0) {
terrno = code;
return terrno;
code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
return TDB_CODE_SUCCESS;
@ -498,28 +491,29 @@ static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream
return TSDB_CODE_SUCCESS;
}
static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) {
static int32_t buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam,
SStreamTask** pAggTask) {
*pAggTask = NULL;
uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
SStreamTask* pAggTask =
int32_t code =
tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5);
if (pAggTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
return pAggTask;
*pTaskList, pStream->conf.fillHistory, pStream->subTableWithoutMd5, pAggTask);
return code;
}
static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, SVgObj* pVgroup,
SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam) {
int32_t code = 0;
SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam);
if (pTask == NULL) {
return terrno;
SStreamTask* pTask = NULL;
code = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam, &pTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pSnode != NULL) {
code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory);
@ -644,14 +638,15 @@ static void bindTwoLevel(SArray* tasks, int32_t begin, int32_t end) {
static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey,
SArray* pVerList) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb;
int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
if (pDbObj == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
bool multiTarget = (pDbObj->cfg.numOfVgroups > 1);
@ -675,9 +670,11 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
SSubplan* plan = getScanSubPlan(pPlan); // source plan
if (plan == NULL) {
return terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
int32_t code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -693,7 +690,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (numOfPlanLevel == 3) {
plan = getAggSubPlan(pPlan, 1); // middle agg plan
if (plan == NULL) {
return terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
do {
SArray** list = taosArrayGetLast(pStream->tasks);
@ -720,7 +719,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
plan = getAggSubPlan(pPlan, 0);
if (plan == NULL) {
return terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
mDebug("doScheduleStream add final agg");
@ -729,7 +730,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
addNewTaskList(pStream);
code = addAggTask(pStream, pMnode, plan, pEpset, true);
if (code != TSDB_CODE_SUCCESS) {
return code;
TAOS_RETURN(code);
}
bindTwoLevel(pStream->tasks, 0, size);
if (pStream->conf.fillHistory) {
@ -740,26 +741,28 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (pStream->conf.fillHistory) {
bindAggSink(pStream, pMnode, pStream->pHTasksList);
}
return TDB_CODE_SUCCESS;
TAOS_RETURN(code);
}
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) {
int32_t code = 0;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
SEpSet mnodeEpset = {0};
mndGetMnodeEpSet(pMnode, &mnodeEpset);
int32_t code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
qDestroyQueryPlan(pPlan);
return code;
TAOS_RETURN(code);
}
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = NULL;
@ -768,21 +771,21 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
SNode* pAst = NULL;
if (nodesStringToNode(pTopic->ast, &pAst) != 0) {
if ((code = nodesStringToNode(pTopic->ast, &pAst)) != 0) {
mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
return -1;
TAOS_RETURN(code);
}
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) {
if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) != 0) {
mError("failed to create topic:%s since %s", pTopic->name, terrstr());
nodesDestroyNode(pAst);
return -1;
TAOS_RETURN(code);
}
nodesDestroyNode(pAst);
}
@ -791,8 +794,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1;
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code);
}
SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
@ -800,8 +803,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1;
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code);
}
pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
@ -836,13 +839,13 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
}
} else {
pSub->qmsg = taosStrdup("");
}
qDestroyQueryPlan(pPlan);
return 0;
TAOS_RETURN(code);
}

54
source/dnode/mnode/impl/test/sdb/sdbTest.cpp
View File

@ -387,11 +387,11 @@ TEST_F(MndTestSdb, 00_API) {
void *pRow2 = sdbGetRowObj(NULL);
ASSERT_EQ(pRow2 == NULL, 1);
//sdbRaw.c
// sdbRaw.c
SStrObj strObj;
SSdbRaw *pRaw1 = NULL;
strSetDefault(&strObj, 1);
pRaw1 = strEncode(&strObj);
int32_t id = sdbGetIdFromRaw(pSdb, pRaw1);
ASSERT_EQ(id, -2);
@ -399,32 +399,32 @@ TEST_F(MndTestSdb, 00_API) {
SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128);
ASSERT_EQ(pRaw2 == NULL, 1);
ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), -1);
ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), -1);
ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), -1);
ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), -1);
ASSERT_EQ(sdbSetRawDataLen(NULL, 0), -1);
ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), -1);
ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), -1);
ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), -1);
ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawDataLen(NULL, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), TSDB_CODE_INVALID_PARA);
ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), -1);
ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), -1);
ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), -1);
ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), -1);
ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), -1);
ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawTotalSize(NULL), -1);
// sdbHash.c

15
source/dnode/mnode/sdb/src/sdb.c
View File

@ -112,6 +112,8 @@ void sdbCleanup(SSdb *pSdb) {
}
int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
int32_t code = 0;
ESdbType sdbType = table.sdbType;
EKeyType keyType = table.keyType;
pSdb->keyTypes[sdbType] = table.keyType;
@ -145,16 +147,17 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
}
static int32_t sdbCreateDir(SSdb *pSdb) {
int32_t code = 0;
if (taosMulMkDir(pSdb->currDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, tstrerror(code));
TAOS_RETURN(code);
}
if (taosMkDir(pSdb->tmpDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->tmpDir, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->tmpDir, tstrerror(code));
TAOS_RETURN(code);
}
return 0;

224
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -14,18 +14,19 @@
*/
#define _DEFAULT_SOURCE
#include "crypt.h"
#include "sdb.h"
#include "sync.h"
#include "tchecksum.h"
#include "wal.h"
#include "tglobal.h"
#include "crypt.h"
#include "wal.h"
#define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512
#define SDB_FILE_VER 1
static int32_t sdbDeployData(SSdb *pSdb) {
int32_t code = 0;
mInfo("start to deploy sdb");
for (int32_t i = SDB_MAX - 1; i >= 0; --i) {
@ -33,8 +34,9 @@ static int32_t sdbDeployData(SSdb *pSdb) {
if (fp == NULL) continue;
mInfo("start to deploy sdb:%s", sdbTableName(i));
if ((*fp)(pSdb->pMnode) != 0) {
mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr());
code = (*fp)(pSdb->pMnode);
if (code != 0) {
mError("failed to deploy sdb:%s since %s", sdbTableName(i), tstrerror(code));
return -1;
}
}
@ -80,61 +82,62 @@ static void sdbResetData(SSdb *pSdb) {
}
static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t code = 0;
int64_t sver = 0;
int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (sver != SDB_FILE_VER) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
ret = taosReadFile(pFile, &pSdb->applyIndex, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
ret = taosReadFile(pFile, &pSdb->applyTerm, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
ret = taosReadFile(pFile, &pSdb->applyConfig, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = 0;
ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (i < SDB_MAX) {
pSdb->maxId[i] = maxId;
@ -145,12 +148,12 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int64_t ver = 0;
ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
if (i < SDB_MAX) {
pSdb->tableVer[i] = ver;
@ -160,37 +163,38 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (ret != sizeof(reserve)) {
terrno = TSDB_CODE_FILE_CORRUPTED;
return -1;
code = TSDB_CODE_FILE_CORRUPTED;
TAOS_RETURN(code);
}
return 0;
}
static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t code = 0;
int64_t sver = SDB_FILE_VER;
if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (taosWriteFile(pFile, &pSdb->applyTerm, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
if (taosWriteFile(pFile, &pSdb->applyConfig, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
@ -199,8 +203,8 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
maxId = pSdb->maxId[i];
}
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
}
@ -210,15 +214,15 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
ver = pSdb->tableVer[i];
}
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
TAOS_RETURN(code);
}
return 0;
@ -237,21 +241,22 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100);
if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since %s", terrstr());
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since %s", tstrerror(code));
TAOS_RETURN(code);
}
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
taosMemoryFree(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno);
mInfo("read sdb file:%s finished since %s", file, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
mInfo("read sdb file:%s finished since %s", file, tstrerror(code));
return 0;
}
if (sdbReadFileHead(pSdb, pFile) != 0) {
mError("failed to read sdb file:%s head since %s", file, terrstr());
code = sdbReadFileHead(pSdb, pFile);
if (code != 0) {
mError("failed to read sdb file:%s head since %s", file, tstrerror(code));
taosMemoryFree(pRaw);
taosCloseFile(&pFile);
return -1;
@ -278,14 +283,14 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
}
readLen = pRaw->dataLen + sizeof(int32_t);
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t);
}
if (readLen >= bufLen) {
bufLen = pRaw->dataLen * 2;
SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100);
if (pNewRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen);
goto _OVER;
}
@ -308,10 +313,14 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
goto _OVER;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
int32_t count = 0;
char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen));
if (plantContent == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
SCryptOpts opts;
opts.len = ENCRYPTED_LEN(pRaw->dataLen);
@ -321,8 +330,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
count = CBC_Decrypt(&opts);
//mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
// mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
memcpy(pRaw->pData, plantContent, pRaw->dataLen);
taosMemoryFree(plantContent);
@ -355,8 +364,7 @@ _OVER:
taosCloseFile(&pFile);
sdbFreeRaw(pRaw);
terrno = code;
return code;
TAOS_RETURN(code);
}
int32_t sdbReadFile(SSdb *pSdb) {
@ -365,7 +373,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
sdbResetData(pSdb);
int32_t code = sdbReadFileImp(pSdb);
if (code != 0) {
mError("failed to read sdb file since %s", terrstr());
mError("failed to read sdb file since %s", tstrerror(code));
sdbResetData(pSdb);
}
@ -388,13 +396,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s for write since %s", tmpfile, terrstr());
return -1;
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s for write since %s", tmpfile, tstrerror(code));
TAOS_RETURN(code);
}
if (sdbWriteFileHead(pSdb, pFile) != 0) {
mError("failed to write sdb file:%s head since %s", tmpfile, terrstr());
code = sdbWriteFileHead(pSdb, pFile);
if (code != 0) {
mError("failed to write sdb file:%s head since %s", tmpfile, tstrerror(code));
taosCloseFile(&pFile);
return -1;
}
@ -436,8 +445,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
int32_t newDataLen = pRaw->dataLen;
char* newData = pRaw->pData;
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
char *newData = pRaw->pData;
if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
newDataLen = ENCRYPTED_LEN(pRaw->dataLen);
newData = taosMemoryMalloc(newDataLen);
if (newData == NULL) {
@ -456,8 +465,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
int32_t count = CBC_Encrypt(&opts);
//mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
// newDataLen, pRaw->dataLen, __FUNCTION__);
// mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
// newDataLen, pRaw->dataLen, __FUNCTION__);
}
if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) {
@ -467,7 +476,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
break;
}
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){
if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
taosMemoryFree(newData);
}
@ -549,19 +558,22 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
}
}
if (code != 0) {
mError("failed to write sdb file since %s", terrstr());
mError("failed to write sdb file since %s", tstrerror(code));
}
taosThreadMutexUnlock(&pSdb->filelock);
return code;
}
int32_t sdbDeploy(SSdb *pSdb) {
if (sdbDeployData(pSdb) != 0) {
return -1;
int32_t code = 0;
code = sdbDeployData(pSdb);
if (code != 0) {
TAOS_RETURN(code);
}
if (sdbWriteFile(pSdb, 0) != 0) {
return -1;
code = sdbWriteFile(pSdb, 0);
if (code != 0) {
TAOS_RETURN(code);
}
return 0;
@ -605,6 +617,7 @@ static void sdbCloseIter(SSdbIter *pIter) {
}
int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config) {
int32_t code = 0;
SSdbIter *pIter = sdbCreateIter(pSdb);
if (pIter == NULL) return -1;
@ -617,19 +630,19 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter
int64_t commitConfig = pSdb->commitConfig;
if (taosCopyFile(datafile, pIter->name) < 0) {
taosThreadMutexUnlock(&pSdb->filelock);
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, tstrerror(code));
sdbCloseIter(pIter);
return -1;
TAOS_RETURN(code);
}
taosThreadMutexUnlock(&pSdb->filelock);
pIter->file = taosOpenFile(pIter->name, TD_FILE_READ);
if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s since %s", pIter->name, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s since %s", pIter->name, tstrerror(code));
sdbCloseIter(pIter);
return -1;
TAOS_RETURN(code);
}
*ppIter = pIter;
@ -645,21 +658,22 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter
void sdbStopRead(SSdb *pSdb, SSdbIter *pIter) { sdbCloseIter(pIter); }
int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
int32_t code = 0;
int32_t maxlen = 4096;
void *pBuf = taosMemoryCalloc(1, maxlen);
if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen);
if (readlen < 0 || readlen > maxlen) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, terrstr(), pIter->total);
code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, tstrerror(code), pIter->total);
*ppBuf = NULL;
*len = 0;
taosMemoryFree(pBuf);
return -1;
TAOS_RETURN(code);
} else if (readlen == 0) {
mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total);
*ppBuf = NULL;
@ -676,15 +690,19 @@ int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
}
int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
int32_t code = 0;
SSdbIter *pIter = sdbCreateIter(pSdb);
if (pIter == NULL) return -1;
if (pIter == NULL) {
code = terrno;
TAOS_RETURN(code);
}
pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open %s since %s", pIter->name, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open %s since %s", pIter->name, tstrerror(code));
sdbCloseIter(pIter);
return -1;
TAOS_RETURN(code);
}
*ppIter = pIter;
@ -702,8 +720,8 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
}
if (taosFsyncFile(pIter->file) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, tstrerror(code));
goto _OVER;
}
@ -713,13 +731,14 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
char datafile[PATH_MAX] = {0};
snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
if (taosRenameFile(pIter->name, datafile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, terrstr());
code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, tstrerror(code));
goto _OVER;
}
if (sdbReadFile(pSdb) != 0) {
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr());
code = sdbReadFile(pSdb);
if (code != 0) {
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, tstrerror(code));
goto _OVER;
}
@ -742,11 +761,12 @@ _OVER:
}
int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) {
int32_t code = 0;
int32_t writelen = taosWriteFile(pIter->file, pBuf, len);
if (writelen != len) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to write len:%d since %s, total:%" PRId64, len, terrstr(), pIter->total);
return -1;
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write len:%d since %s, total:%" PRId64, len, tstrerror(code), pIter->total);
TAOS_RETURN(code);
}
pIter->total += writelen;

131
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -53,14 +53,15 @@ void sdbFreeRaw(SSdbRaw *pRaw) {
}
int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*(int8_t *)(pRaw->pData + dataPos) = val;
@ -68,14 +69,15 @@ int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
}
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*(uint8_t *)(pRaw->pData + dataPos) = val;
@ -83,14 +85,15 @@ int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
}
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int32_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*(int32_t *)(pRaw->pData + dataPos) = val;
@ -98,14 +101,15 @@ int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
}
int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int16_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*(int16_t *)(pRaw->pData + dataPos) = val;
@ -113,14 +117,15 @@ int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) {
}
int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int64_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*(int64_t *)(pRaw->pData + dataPos) = val;
@ -128,14 +133,15 @@ int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) {
}
int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + valLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
if (pVal != NULL) {
@ -145,14 +151,15 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
}
int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
pRaw->dataLen = dataLen;
@ -160,14 +167,15 @@ int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) {
}
int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (status == SDB_STATUS_INIT) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
code = TSDB_CODE_INVALID_PARA;
TAOS_RETURN(code);
}
pRaw->status = status;
@ -175,14 +183,15 @@ int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) {
}
int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*val = *(int8_t *)(pRaw->pData + dataPos);
@ -190,14 +199,15 @@ int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
}
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*val = *(uint8_t *)(pRaw->pData + dataPos);
@ -205,14 +215,15 @@ int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
}
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int32_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*val = *(int32_t *)(pRaw->pData + dataPos);
@ -220,14 +231,15 @@ int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
}
int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int16_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*val = *(int16_t *)(pRaw->pData + dataPos);
@ -235,14 +247,15 @@ int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) {
}
int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + sizeof(int64_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
*val = *(int64_t *)(pRaw->pData + dataPos);
@ -250,14 +263,15 @@ int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) {
}
int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
if (dataPos + valLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
code = TSDB_CODE_SDB_INVALID_DATA_LEN;
TAOS_RETURN(code);
}
if (pVal != NULL) {
memcpy(pVal, pRaw->pData + dataPos, valLen);
@ -266,9 +280,10 @@ int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valL
}
int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver) {
int32_t code = 0;
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
*sver = pRaw->sver;

16
source/dnode/snode/src/snode.c
View File

@ -41,7 +41,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
SCheckpointInfo *pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask);
char *p = streamTaskGetStatus(pTask)->name;
char *p = streamTaskGetStatus(pTask).name;
if (pTask->info.fillHistory) {
sndInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
@ -61,6 +61,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
}
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
int32_t code = 0;
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -68,12 +69,17 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
}
stopRsync();
startRsync();
code = startRsync();
if (code != 0) {
terrno = code;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
if (pSnode->pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE,
taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto FAIL;
}

5
source/dnode/vnode/CMakeLists.txt
View File

@ -68,11 +68,8 @@ set(
"src/tq/tqOffset.c"
"src/tq/tqPush.c"
"src/tq/tqSink.c"
"src/tq/tqCommit.c"
"src/tq/tqStreamTask.c"
"src/tq/tqHandleSnapshot.c"
"src/tq/tqCheckInfoSnapshot.c"
"src/tq/tqOffsetSnapshot.c"
"src/tq/tqSnapshot.c"
"src/tq/tqStreamStateSnap.c"
"src/tq/tqStreamTaskSnap.c"

51
source/dnode/vnode/src/inc/tq.h
View File

@ -41,8 +41,6 @@ extern "C" {
#define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
// clang-format on
typedef struct STqOffsetStore STqOffsetStore;
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
// tqExec
@ -101,10 +99,11 @@ struct STQ {
SHashObj* pPushMgr; // subKey -> STqHandle
SHashObj* pHandle; // subKey -> STqHandle
SHashObj* pCheckInfo; // topic -> SAlterCheckInfo
STqOffsetStore* pOffsetStore;
SHashObj* pOffset; // subKey -> STqOffsetVal
TDB* pMetaDB;
TTB* pExecStore;
TTB* pCheckStore;
TTB* pOffsetStore;
SStreamMeta* pStreamMeta;
};
@ -128,29 +127,22 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
int32_t tqMetaOpen(STQ* pTq);
int32_t tqMetaClose(STQ* pTq);
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
int32_t tqMetaRestoreCheckInfo(STQ* pTq);
int32_t tqMetaGetHandle(STQ* pTq, const char* key);
int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle);
STqOffsetStore* tqOffsetOpen(STQ* pTq);
int32_t tqMetaTransform(STQ* pTq);
void tqOffsetClose(STqOffsetStore*);
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset);
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetCommitFile(STqOffsetStore* pStore);
int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen);
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen);
int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle);
int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen);
int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen);
int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle);
void* tqMetaGetOffset(STQ* pTq, const char* subkey);
int32_t tqMetaTransform(STQ* pTq);
// tqSink
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule);
void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data);
// tqOffset
char* tqOffsetBuildFName(const char* path, int32_t fVer);
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
int32_t tqBuildFName(char** data, const char* path, char* name);
int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name);
// tq util
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
@ -166,6 +158,25 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule);
#define TQ_ERR_GO_TO_END(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
goto END; \
} \
} while (0)
#define TQ_ERR_RETURN(c) \
do { \
code = c; \
if (code != TSDB_CODE_SUCCESS) { \
return code; \
} \
} while (0)
#define TQ_SUBSCRIBE_NAME "subscribe"
#define TQ_OFFSET_NAME "offset-ver0"
#ifdef __cplusplus
}
#endif

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -258,7 +258,6 @@ int32_t tsdbGetNRowsInTbData(STbData *pTbData);
typedef enum { TSDB_HEAD_FILE = 0, TSDB_DATA_FILE, TSDB_LAST_FILE, TSDB_SMA_FILE } EDataFileT;
bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);

33
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -70,10 +70,6 @@ typedef struct STsdbSnapRAWReader STsdbSnapRAWReader;
typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter;
typedef struct STqSnapReader STqSnapReader;
typedef struct STqSnapWriter STqSnapWriter;
typedef struct STqOffsetReader STqOffsetReader;
typedef struct STqOffsetWriter STqOffsetWriter;
typedef struct STqCheckInfoReader STqCheckInfoReader;
typedef struct STqCheckInfoWriter STqCheckInfoWriter;
typedef struct SStreamTaskReader SStreamTaskReader;
typedef struct SStreamTaskWriter SStreamTaskWriter;
typedef struct SStreamStateReader SStreamStateReader;
@ -222,8 +218,8 @@ typedef struct SMetaInfo {
int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
// tsdb
int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force);
int tsdbClose(STsdb** pTsdb);
int32_t tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force);
int32_t tsdbClose(STsdb** pTsdb);
int32_t tsdbBegin(STsdb* pTsdb);
// int32_t tsdbPrepareCommit(STsdb* pTsdb);
// int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
@ -266,7 +262,6 @@ int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq);
int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId);
// tq-mq
@ -349,29 +344,17 @@ int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr);
int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter);
int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback);
// STqSnapshotReader ==
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader);
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader);
int32_t tqSnapReaderClose(STqSnapReader** ppReader);
int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData);
// STqSnapshotWriter ======================================
int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter);
int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback);
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
// STqCheckInfoshotReader ==
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader);
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader);
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData);
// STqCheckInfoshotWriter ======================================
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter);
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback);
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData);
// STqOffsetReader ========================================
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader);
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader);
int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData);
// STqOffsetWriter ========================================
int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter);
int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback);
int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
// SStreamTaskWriter ======================================
int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader);

19
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -297,8 +297,17 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
sprintf(pStreamTask->exec.qmsg, "%s", RSMA_EXEC_TASK_FLAG);
pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta);
tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask);
pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo();
int32_t code = streamCreateStateMachine(pStreamTask);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = streamTaskCreateActiveChkptInfo(&pStreamTask->chkInfo.pActiveInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId);
if (!pStreamState) {
terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
@ -414,7 +423,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
goto _err;
}
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != 0) {
goto _err;
}
@ -540,12 +549,12 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid)
taosArrayDestroy(pUidArray);
return TSDB_CODE_FAILED;
}
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
return TSDB_CODE_FAILED;
}
}
} else {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) < 0) {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
return TSDB_CODE_FAILED;
}
}

129
source/dnode/vnode/src/tq/tq.c
View File

@ -76,9 +76,12 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
taosInitRWLatch(&pTq->lock);
pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK);
pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
pTq->pCheckInfo = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo);
pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pOffset, (FDelete)tDeleteSTqOffset);
int32_t code = tqInitialize(pTq);
if (code != TSDB_CODE_SUCCESS) {
tqClose(pTq);
@ -90,26 +93,19 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
int32_t tqInitialize(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode);
pTq->pStreamMeta =
streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback);
if (pTq->pStreamMeta == NULL) {
return -1;
int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1,
tqStartTaskCompleteCallback, &pTq->pStreamMeta);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
streamMetaLoadAllTasks(pTq->pStreamMeta);
if (tqMetaTransform(pTq) < 0) {
if (tqMetaOpen(pTq) < 0) {
return -1;
}
if (tqMetaRestoreCheckInfo(pTq) < 0) {
return -1;
}
pTq->pOffsetStore = tqOffsetOpen(pTq);
if (pTq->pOffsetStore == NULL) {
return -1;
}
return 0;
}
@ -133,10 +129,10 @@ void tqClose(STQ* pTq) {
pIter = taosHashIterate(pTq->pPushMgr, pIter);
}
tqOffsetClose(pTq->pOffsetStore);
taosHashCleanup(pTq->pHandle);
taosHashCleanup(pTq->pPushMgr);
taosHashCleanup(pTq->pCheckInfo);
taosHashCleanup(pTq->pOffset);
taosMemoryFree(pTq->path);
tqMetaClose(pTq);
@ -221,7 +217,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
goto end;
}
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey);
STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, pOffset->subKey);
if (pSavedOffset != NULL && tqOffsetEqual(pOffset, pSavedOffset)) {
tqInfo("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64,
vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version);
@ -229,10 +225,14 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
}
// save the new offset value
code = tqOffsetWrite(pTq->pOffsetStore, pOffset);
if(code != 0) {
code = TSDB_CODE_INVALID_MSG;
goto end;
if (taosHashPut(pTq->pOffset, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset))){
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), msg, msgLen - sizeof(vgOffset.consumerId)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
@ -360,21 +360,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
while (1) {
taosWLockLatch(&pTq->lock);
// 1. find handle
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle == NULL) {
do {
if (tqMetaGetHandle(pTq, req.subKey) == 0) {
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle != NULL) {
break;
}
}
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG;
taosWUnLockLatch(&pTq->lock);
code = -1;
goto END;
} while (0);
code = tqMetaGetHandle(pTq, req.subKey, &pHandle);
if (code != TDB_CODE_SUCCESS) {
tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
terrno = TSDB_CODE_INVALID_MSG;
taosWUnLockLatch(&pTq->lock);
return -1;
}
// 2. check rebalance status
@ -443,7 +434,7 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) {
tDecoderClear(&decoder);
STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, vgOffset.offset.subKey);
STqOffset* pSavedOffset = (STqOffset*)tqMetaGetOffset(pTq, vgOffset.offset.subKey);
if (pSavedOffset == NULL) {
terrno = TSDB_CODE_TMQ_NO_COMMITTED;
return terrno;
@ -523,7 +514,7 @@ int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) {
if (reqOffset.type == TMQ_OFFSET__LOG) {
dataRsp.common.rspOffset.version = reqOffset.version;
} else if (reqOffset.type < 0) {
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey);
STqOffset* pOffset = (STqOffset*)(STqOffset*)tqMetaGetOffset(pTq, req.subKey);
if (pOffset != NULL) {
if (pOffset->val.type != TMQ_OFFSET__LOG) {
tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s, no valid wal info", consumerId, vgId, req.subKey);
@ -590,12 +581,14 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
}
taosWLockLatch(&pTq->lock);
code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey);
if (code != 0) {
tqError("cannot process tq delete req %s, since no such offset in cache", pReq->subKey);
if (taosHashRemove(pTq->pOffset, pReq->subKey, strlen(pReq->subKey)) != 0) {
tqError("cannot process tq delete req %s, since no such offset in hash", pReq->subKey);
}
if (tqMetaDeleteInfo(pTq, pTq->pOffsetStore, pReq->subKey, strlen(pReq->subKey)) != 0) {
tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey);
}
if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) {
if (tqMetaDeleteInfo(pTq, pTq->pExecStore, pReq->subKey, strlen(pReq->subKey)) < 0) {
tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey);
}
taosWUnLockLatch(&pTq->lock);
@ -605,18 +598,16 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
STqCheckInfo info = {0};
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
if (tDecodeSTqCheckInfo(&decoder, &info) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
if(tqMetaDecodeCheckInfo(&info, msg, msgLen) != 0){
return -1;
}
tDecoderClear(&decoder);
if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSTqCheckInfo(&info);
return -1;
}
if (tqMetaSaveCheckInfo(pTq, info.topic, msg, msgLen) < 0) {
if (tqMetaSaveInfo(pTq, pTq->pCheckStore, info.topic, strlen(info.topic), msg, msgLen) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
@ -628,7 +619,8 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (tqMetaDeleteCheckInfo(pTq, msg) < 0) {
if (tqMetaDeleteInfo(pTq, pTq->pCheckStore, msg, strlen(msg)) < 0) {
tqError("cannot process tq delete check info req %s, since no such check info", msg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
@ -652,19 +644,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey,
req.oldConsumerId, req.newConsumerId);
taosRLockLatch(&pTq->lock);
STqHandle* pHandle = NULL;
while (1) {
pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
if (pHandle) {
break;
}
taosRLockLatch(&pTq->lock);
ret = tqMetaGetHandle(pTq, req.subKey);
taosRUnLockLatch(&pTq->lock);
if (ret < 0) {
break;
}
}
(void)tqMetaGetHandle(pTq, req.subKey, &pHandle); //ignore return code
taosRUnLockLatch(&pTq->lock);
if (pHandle == NULL) {
if (req.oldConsumerId != -1) {
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
@ -675,7 +658,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
goto end;
}
STqHandle handle = {0};
ret = tqCreateHandle(pTq, &req, &handle);
ret = tqMetaCreateHandle(pTq, &req, &handle);
if (ret < 0) {
tqDestroyTqHandle(&handle);
goto end;
@ -767,7 +750,7 @@ int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessV
SCheckpointInfo* pChkInfo = &pTask->chkInfo;
tqSetRestoreVersionInfo(pTask);
char* p = streamTaskGetStatus(pTask)->name;
char* p = streamTaskGetStatus(pTask).name;
const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus);
if (pTask->info.fillHistory) {
@ -856,7 +839,8 @@ int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) {
SStreamMeta* pMeta = pStreamTask->pMeta;
STaskId hId = pStreamTask->hTaskInfo.id;
SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId);
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId, &pTask);
if (pTask == NULL) {
tqWarn("s-task:0x%x failed to acquired it to exec step 2, scan wal quit", (int32_t) hId.taskId);
return TSDB_CODE_SUCCESS;
@ -874,7 +858,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = TSDB_CODE_SUCCESS;
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) {
tqError("vgId:%d failed to acquire stream task:0x%x during scan history data, task may have been destroyed",
pMeta->vgId, pReq->taskId);
@ -883,7 +868,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
// do recovery step1
const char* id = pTask->id.idStr;
char* pStatus = streamTaskGetStatus(pTask)->name;
char* pStatus = streamTaskGetStatus(pTask).name;
// avoid multi-thread exec
while (1) {
@ -939,15 +924,15 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
if (retInfo.ret == TASK_SCANHISTORY_REXEC) {
streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else {
SStreamTaskState* p = streamTaskGetStatus(pTask);
ETaskStatus s = p->state;
SStreamTaskState p = streamTaskGetStatus(pTask);
ETaskStatus s = p.state;
if (s == TASK_STATUS__PAUSE) {
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr,
el, pTask->execInfo.step1El, status);
} else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) {
tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr,
p->name, pTask->execInfo.step1El);
p.name, pTask->execInfo.step1El);
}
}
@ -961,7 +946,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
ASSERT(pTask->info.fillHistory == 1);
// 1. get the related stream task
SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
SStreamTask* pStreamTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId, &pStreamTask);
if (pStreamTask == NULL) {
tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
pTask->streamTaskId.taskId, pTask->id.idStr);
@ -1121,7 +1107,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS;
}
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) {
tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64
" transId:%d it may have been destroyed",
@ -1147,7 +1134,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
// todo save the checkpoint failed info
taosThreadMutexLock(&pTask->lock);
ETaskStatus status = streamTaskGetStatus(pTask)->state;
ETaskStatus status = streamTaskGetStatus(pTask).state;
if (req.mndTrigger == 1) {
if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {

196
source/dnode/vnode/src/tq/tqCheckInfoSnapshot.c
View File

@ -1,196 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
#include "tdbInt.h"
#include "tq.h"
// STqCheckInfoReader ========================================
struct STqCheckInfoReader {
STQ* pTq;
int64_t sver;
int64_t ever;
TBC* pCur;
};
int32_t tqCheckInfoReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoReader** ppReader) {
int32_t code = 0;
STqCheckInfoReader* pReader = NULL;
// alloc
pReader = (STqCheckInfoReader*)taosMemoryCalloc(1, sizeof(STqCheckInfoReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
// impl
code = tdbTbcOpen(pTq->pCheckStore, &pReader->pCur, NULL);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
code = tdbTbcMoveToFirst(pReader->pCur);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
tqInfo("vgId:%d, vnode checkinfo tq reader opened", TD_VID(pTq->pVnode));
*ppReader = pReader;
return code;
_err:
tqError("vgId:%d, vnode checkinfo tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppReader = NULL;
return code;
}
int32_t tqCheckInfoReaderClose(STqCheckInfoReader** ppReader) {
int32_t code = 0;
tdbTbcClose((*ppReader)->pCur);
taosMemoryFree(*ppReader);
*ppReader = NULL;
return code;
}
int32_t tqCheckInfoRead(STqCheckInfoReader* pReader, uint8_t** ppData) {
int32_t code = 0;
void* pKey = NULL;
void* pVal = NULL;
int32_t kLen = 0;
int32_t vLen = 0;
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
goto _exit;
}
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
pHdr->type = SNAP_DATA_TQ_CHECKINFO;
pHdr->size = vLen;
memcpy(pHdr->data, pVal, vLen);
_exit:
tdbFree(pKey);
tdbFree(pVal);
tqInfo("vgId:%d, vnode check info tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
return code;
_err:
tdbFree(pKey);
tdbFree(pVal);
tqError("vgId:%d, vnode check info tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
return code;
}
// STqCheckInfoWriter ========================================
struct STqCheckInfoWriter {
STQ* pTq;
int64_t sver;
int64_t ever;
TXN* txn;
};
int32_t tqCheckInfoWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqCheckInfoWriter** ppWriter) {
int32_t code = 0;
STqCheckInfoWriter* pWriter;
// alloc
pWriter = (STqCheckInfoWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pWriter->pTq = pTq;
pWriter->sver = sver;
pWriter->ever = ever;
if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
code = -1;
taosMemoryFree(pWriter);
goto _err;
}
*ppWriter = pWriter;
return code;
_err:
tqError("vgId:%d, tq check info writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
int32_t tqCheckInfoWriterClose(STqCheckInfoWriter** ppWriter, int8_t rollback) {
int32_t code = 0;
STqCheckInfoWriter* pWriter = *ppWriter;
STQ* pTq = pWriter->pTq;
if (rollback) {
tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
} else {
code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err;
code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err;
}
taosMemoryFree(pWriter);
*ppWriter = NULL;
return code;
_err:
tqError("vgId:%d, tq check info writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}
int32_t tqCheckInfoWrite(STqCheckInfoWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqCheckInfo info = {0};
SDecoder decoder;
SDecoder* pDecoder = &decoder;
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = tDecodeSTqCheckInfo(pDecoder, &info);
if (code) goto _err;
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
if (code) goto _err;
code = tqMetaSaveCheckInfo(pTq, info.topic, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
if (code) goto _err;
tDecoderClear(pDecoder);
return code;
_err:
tDecoderClear(pDecoder);
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}

554
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -75,125 +75,98 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
return 0;
}
int32_t tqMetaOpen(STQ* pTq) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL) < 0) {
return -1;
int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen){
SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqCheckInfo(&decoder, info);
if (code != 0) {
tDeleteSTqCheckInfo(info);
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) {
return -1;
}
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaClose(STQ* pTq) {
if (pTq->pExecStore) {
tdbTbClose(pTq->pExecStore);
}
if (pTq->pCheckStore) {
tdbTbClose(pTq->pCheckStore);
}
tdbClose(pTq->pMetaDB);
return 0;
}
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) {
return -1;
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) {
tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key);
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) {
return -1;
}
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
SDecoder decoder;
int32_t code = 0;
tdbTbcMoveToFirst(pCur);
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
STqCheckInfo info;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
code = tDecodeSTqCheckInfo(&decoder, &info);
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
tDecoderClear(&decoder);
code = taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo));
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}
END:
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
tDecoderClear(&decoder);
return code;
}
int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen){
SDecoder decoder = {0};
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
int32_t code = tDecodeSTqOffset(&decoder, info);
if (code != 0) {
tDeleteSTqOffset(info);
return TSDB_CODE_OUT_OF_MEMORY;
}
tDecoderClear(&decoder);
return code;
}
int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
}
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) {
int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL;
TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbDelete(ttb, key, kLen, txn));
TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0;
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
}
void* tqMetaGetOffset(STQ* pTq, const char* subkey){
void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey));
if (data == NULL) {
int vLen = 0;
if (tdbTbGet(pTq->pOffsetStore, subkey, strlen(subkey), &data, &vLen) < 0) {
tdbFree(data);
return NULL;
}
STqOffset offset = {0};
if (tqMetaDecodeOffsetInfo(&offset, data, vLen) != TDB_CODE_SUCCESS) {
tdbFree(data);
return NULL;
}
if(taosHashPut(pTq->pOffset, subkey, strlen(subkey), &offset, sizeof(STqOffset)) != 0){
tDeleteSTqOffset(&offset);
tdbFree(data);
return NULL;
}
tdbFree(data);
return taosHashGet(pTq->pOffset, subkey, strlen(subkey));
} else {
return data;
}
}
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
int32_t code;
int32_t code = TDB_CODE_SUCCESS;
int32_t vlen;
void* buf = NULL;
SEncoder encoder;
tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
if (code < 0) {
goto end;
goto END;
}
tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 " epoch:%d vgId:%d", pHandle->subKey,
@ -202,71 +175,33 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
buf = taosMemoryCalloc(1, vlen);
if (buf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
goto END;
}
tEncoderInit(&encoder, buf, vlen);
code = tEncodeSTqHandle(&encoder, pHandle);
if (code < 0) {
goto end;
goto END;
}
TXN* txn = NULL;
code = tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
if (code < 0) {
goto end;
}
TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pExecStore, key, (int)strlen(key), buf, vlen));
code = tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn);
if (code < 0) {
goto end;
}
code = tdbCommit(pTq->pMetaDB, txn);
if (code < 0) {
goto end;
}
code = tdbPostCommit(pTq->pMetaDB, txn);
end:
END:
tEncoderClear(&encoder);
taosMemoryFree(buf);
return code;
}
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) {
return -1;
}
if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) {
}
if (tdbCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
return -1;
}
return 0;
}
static int buildHandle(STQ* pTq, STqHandle* handle){
static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
int32_t code = TDB_CODE_SUCCESS;
SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode);
handle->pRef = walOpenRef(pVnode->pWal);
if (handle->pRef == NULL) {
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
walSetRefVer(handle->pRef, handle->snapshotVer);
TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer));
SReadHandle reader = {
.vnode = pVnode,
@ -282,18 +217,18 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId);
if (handle->execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle->subKey);
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
void* scanner = NULL;
qExtractStreamScanner(handle->execHandle.task, &scanner);
(void)qExtractStreamScanner(handle->execHandle.task, &scanner);
if (scanner == NULL) {
tqError("cannot extract stream scanner for %s", handle->subKey);
return -1;
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
handle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner);
if (handle->execHandle.pTqReader == NULL) {
tqError("cannot extract exec reader for %s", handle->subKey);
return -1;
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
} else if (handle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
handle->pWalReader = walOpenReader(pVnode->pWal, NULL, 0);
@ -308,7 +243,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
if(handle->execHandle.execTb.qmsg != NULL && strcmp(handle->execHandle.execTb.qmsg, "") != 0) {
if (nodesStringToNode(handle->execHandle.execTb.qmsg, &handle->execHandle.execTb.node) != 0) {
tqError("nodesStringToNode error in sub stable, since %s", terrstr());
return -1;
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
}
buildSnapContext(reader.vnode, reader.version, handle->execHandle.execTb.suid, handle->execHandle.subType,
@ -320,7 +255,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
if(ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId);
taosArrayDestroy(tbUidList);
return -1;
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
handle->execHandle.pTqReader = tqReaderOpen(pVnode);
@ -330,24 +265,23 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
return 0;
}
static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){
static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode);
SDecoder decoder;
int32_t code = 0;
SDecoder decoder = {0};
int32_t code = TDB_CODE_SUCCESS;
tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
code = tDecodeSTqHandle(&decoder, handle);
if (code) goto end;
code = buildHandle(pTq, handle);
if (code) goto end;
tqInfo("restoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
TQ_ERR_GO_TO_END(tDecodeSTqHandle(&decoder, handle));
TQ_ERR_GO_TO_END(tqMetaInitHandle(pTq, handle));
tqInfo("tqMetaRestoreHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
code = taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
end:
END:
tDecoderClear(&decoder);
return code;
}
int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
int32_t vgId = TD_VID(pTq->pVnode);
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
@ -367,165 +301,189 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal);
if(buildHandle(pTq, handle) < 0){
if(tqMetaInitHandle(pTq, handle) < 0){
return -1;
}
tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
}
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pExecStoreOld, TTB* pExecStoreNew){
TBC* pCur = NULL;
if (tdbTbcOpen(pExecStoreOld, &pCur, NULL) < 0) {
return -1;
}
TXN* txn;
if (tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
return -1;
}
static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
TXN* txn = NULL;
tdbTbcMoveToFirst(pCur);
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
if (tdbTbUpsert(pExecStoreNew, pKey, kLen, pVal, vLen, txn) < 0) {
tqError("transform sub info error");
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
return -1;
}
TQ_ERR_GO_TO_END (tdbTbUpsert(pNew, pKey, kLen, pVal, vLen, txn));
}
TQ_ERR_GO_TO_END (tdbCommit(pMetaDB, txn));
TQ_ERR_GO_TO_END (tdbPostCommit(pMetaDB, txn));
END:
tdbFree(pKey);
tdbFree(pVal);
(void)tdbTbcClose(pCur);
return code;
}
int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) {
void* data = taosHashGet(pTq->pHandle, key, strlen(key));
if(data == NULL){
int vLen = 0;
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &data, &vLen) < 0) {
tdbFree(data);
return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST;
}
STqHandle handle = {0};
if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0){
tdbFree(data);
tqDestroyTqHandle(&handle);
return TSDB_CODE_OUT_OF_MEMORY;
}
tdbFree(data);
*pHandle = taosHashGet(pTq->pHandle, key, strlen(key));
if(*pHandle == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
*pHandle = data;
}
return TDB_CODE_SUCCESS;
}
int32_t tqMetaOpenTdb(STQ* pTq) {
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.offset.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pOffsetStore, 0));
END:
return code;
}
static int32_t replaceTqPath(char** path){
char* tpath = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME));
taosMemoryFree(*path);
*path = tpath;
return TDB_CODE_SUCCESS;
}
static int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL;
void* pKey = NULL;
int kLen = 0;
void* pVal = NULL;
int vLen = 0;
int32_t code = 0;
STqCheckInfo info = {0};
TQ_ERR_GO_TO_END(tdbTbcOpen(pTq->pCheckStore, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen));
TQ_ERR_GO_TO_END(taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)));
}
info.colIdList = NULL;
END:
tdbFree(pKey);
tdbFree(pVal);
tdbTbcClose(pCur);
tDeleteSTqCheckInfo(&info);
return code;
}
if (tdbCommit(pMetaDB, txn) < 0) {
return -1;
int32_t tqMetaOpen(STQ* pTq) {
char* maindb = NULL;
char* offsetNew = NULL;
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){
TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{
TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
(void)taosRemoveFile(maindb);
}
if (tdbPostCommit(pMetaDB, txn) < 0) {
return -1;
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offsetNew)){
TQ_ERR_GO_TO_END(tqOffsetRestoreFromFile(pTq, offsetNew));
(void)taosRemoveFile(offsetNew);
}
return 0;
TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq));
END:
taosMemoryFree(maindb);
taosMemoryFree(offsetNew);
return code;
}
int32_t tqMetaTransform(STQ* pTq) {
int32_t len = strlen(pTq->path) + 64;
char* maindb = taosMemoryCalloc(1, len);
sprintf(maindb, "%s%s%s", pTq->path, TD_DIRSEP, TDB_MAINDB_NAME);
int32_t code = TDB_CODE_SUCCESS;
TDB* pMetaDB = NULL;
TTB* pExecStore = NULL;
TTB* pCheckStore = NULL;
char* offsetNew = NULL;
char* offset = NULL;
TQ_ERR_GO_TO_END(tqBuildFName(&offset, pTq->path, TQ_OFFSET_NAME));
if(!taosCheckExistFile(maindb)){
taosMemoryFree(maindb);
char* tpath = taosMemoryCalloc(1, len);
if(tpath == NULL){
return -1;
TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0));
TQ_ERR_GO_TO_END(tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0));
TQ_ERR_GO_TO_END(replaceTqPath(&pTq->path));
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore));
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offset)) {
if (taosCopyFile(offset, offsetNew) < 0) {
tqError("copy offset file error");
} else {
(void)taosRemoveFile(offset);
}
sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe");
taosMemoryFree(pTq->path);
pTq->path = tpath;
return tqMetaOpen(pTq);
}
int32_t code = 0;
TDB* pMetaDB = NULL;
TTB* pExecStore = NULL;
TTB* pCheckStore = NULL;
char* offsetNew = NULL;
char* offset = tqOffsetBuildFName(pTq->path, 0);
if(offset == NULL){
code = -1;
goto END;
}
if (tdbOpen(pTq->path, 16 * 1024, 1, &pMetaDB, 0, 0, NULL) < 0) {
code = -1;
goto END;
}
if (tdbTbOpen("tq.db", -1, -1, NULL, pMetaDB, &pExecStore, 0) < 0) {
code = -1;
goto END;
}
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pMetaDB, &pCheckStore, 0) < 0) {
code = -1;
goto END;
}
char* tpath = taosMemoryCalloc(1, len);
if(tpath == NULL){
code = -1;
goto END;
}
sprintf(tpath, "%s%s%s", pTq->path, TD_DIRSEP, "subscribe");
taosMemoryFree(pTq->path);
pTq->path = tpath;
if (tqMetaOpen(pTq) < 0) {
code = -1;
goto END;
}
if( tqMetaTransformInfo(pTq->pMetaDB, pExecStore, pTq->pExecStore) < 0){
code = -1;
goto END;
}
if(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore) < 0){
code = -1;
goto END;
}
tdbTbClose(pExecStore);
pExecStore = NULL;
tdbTbClose(pCheckStore);
pCheckStore = NULL;
tdbClose(pMetaDB);
pMetaDB = NULL;
offsetNew = tqOffsetBuildFName(pTq->path, 0);
if(offsetNew == NULL){
code = -1;
goto END;
}
if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){
tqError("copy offset file error");
code = -1;
goto END;
}
taosRemoveFile(maindb);
taosRemoveFile(offset);
END:
taosMemoryFree(maindb);
END:
taosMemoryFree(offset);
taosMemoryFree(offsetNew);
tdbTbClose(pExecStore);
tdbTbClose(pCheckStore);
tdbClose(pMetaDB);
//return 0 always, so ignore
(void)tdbTbClose(pExecStore);
(void)tdbTbClose(pCheckStore);
(void)tdbClose(pMetaDB);
return code;
}
int32_t tqMetaGetHandle(STQ* pTq, const char* key) {
void* pVal = NULL;
int vLen = 0;
if (tdbTbGet(pTq->pExecStore, key, (int)strlen(key), &pVal, &vLen) < 0) {
return -1;
int32_t tqMetaClose(STQ* pTq) {
if (pTq->pExecStore) {
(void)tdbTbClose(pTq->pExecStore);
}
STqHandle handle = {0};
int code = restoreHandle(pTq, pVal, vLen, &handle);
if (code < 0){
tqDestroyTqHandle(&handle);
if (pTq->pCheckStore) {
(void)tdbTbClose(pTq->pCheckStore);
}
tdbFree(pVal);
return code;
}
if (pTq->pOffsetStore) {
(void)tdbTbClose(pTq->pOffsetStore);
}
(void)tdbClose(pTq->pMetaDB);
return 0;
}

205
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -16,192 +16,71 @@
#include "tq.h"
struct STqOffsetStore {
STQ* pTq;
SHashObj* pHash; // SHashObj<subscribeKey, offset>
int8_t needCommit;
};
char* tqOffsetBuildFName(const char* path, int32_t fVer) {
int32_t len = strlen(path);
char* fname = taosMemoryCalloc(1, len + 40);
int32_t tqBuildFName(char** data, const char* path, char* name) {
int32_t len = strlen(path) + strlen(name) + 2;
char* fname = taosMemoryCalloc(1, len);
if(fname == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
snprintf(fname, len + 40, "%s/offset-ver%d", path, fVer);
return fname;
int32_t code = snprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name);
if (code < 0){
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(fname);
return code;
}
*data = fname;
return TDB_CODE_SUCCESS;
}
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
int32_t code = TDB_CODE_SUCCESS;
void* pMemBuf = NULL;
TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ);
if (pFile == NULL) {
return TSDB_CODE_SUCCESS;
code = TDB_CODE_SUCCESS;
goto END;
}
int32_t vgId = TD_VID(pStore->pTq->pVnode);
int64_t code = 0;
int64_t ret = 0;
int32_t size = 0;
while (1) {
if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) {
if (code == 0) {
break;
} else {
return -1;
if ((ret = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) {
if (ret != 0) {
code = TSDB_CODE_INVALID_MSG;
}
goto END;
}
size = htonl(size);
void* pMemBuf = taosMemoryCalloc(1, size);
pMemBuf = taosMemoryCalloc(1, size);
if (pMemBuf == NULL) {
tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
if ((code = taosReadFile(pFile, pMemBuf, size)) != size) {
taosMemoryFree(pMemBuf);
return -1;
if (taosReadFile(pFile, pMemBuf, size) != size) {
terrno = TSDB_CODE_INVALID_MSG;
goto END;
}
STqOffset offset = {0};
SDecoder decoder;
tDecoderInit(&decoder, pMemBuf, size);
if (tDecodeSTqOffset(&decoder, &offset) < 0) {
taosMemoryFree(pMemBuf);
tDecoderClear(&decoder);
return code;
}
tDecoderClear(&decoder);
if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) {
return -1;
}
// todo remove this
if (offset.val.type == TMQ_OFFSET__LOG) {
STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey));
if (pHandle) {
if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) {
// tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, pHandle->subKey,
// offset.val.version);
}
}
TQ_ERR_GO_TO_END(tqMetaDecodeOffsetInfo(&offset, pMemBuf, size));
code = taosHashPut(pTq->pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset));
if (code != TDB_CODE_SUCCESS) {
tDeleteSTqOffset(&offset);
goto END;
}
TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size));
tqInfo("tq: offset restore from file to tdb, subkey:%s", offset.subKey);
taosMemoryFree(pMemBuf);
pMemBuf = NULL;
}
END:
taosCloseFile(&pFile);
return TSDB_CODE_SUCCESS;
}
STqOffsetStore* tqOffsetOpen(STQ* pTq) {
STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore));
if (pStore == NULL) {
return NULL;
}
pStore->pTq = pTq;
pStore->needCommit = 0;
pTq->pOffsetStore = pStore;
pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
if (pStore->pHash == NULL) {
taosMemoryFree(pStore);
return NULL;
}
taosHashSetFreeFp(pStore->pHash, tOffsetDestroy);
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
if (tqOffsetRestoreFromFile(pStore, fname) < 0) {
taosMemoryFree(fname);
taosMemoryFree(pStore);
return NULL;
}
taosMemoryFree(fname);
return pStore;
}
void tqOffsetClose(STqOffsetStore* pStore) {
if(pStore == NULL) return;
tqOffsetCommitFile(pStore);
taosHashCleanup(pStore->pHash);
taosMemoryFree(pStore);
}
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) {
return (STqOffset*)taosHashGet(pStore->pHash, subscribeKey, strlen(subscribeKey));
}
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) {
pStore->needCommit = 1;
return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
}
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) {
return taosHashRemove(pStore->pHash, subscribeKey, strlen(subscribeKey));
}
int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
if (!pStore->needCommit) {
return 0;
}
// TODO file name should be with a newer version
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
const char* err = strerror(errno);
tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);
void* pIter = NULL;
while (1) {
pIter = taosHashIterate(pStore->pHash, pIter);
if (pIter == NULL) {
break;
}
STqOffset* pOffset = (STqOffset*)pIter;
int32_t bodyLen;
int32_t code;
tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code);
if (code < 0) {
taosHashCancelIterate(pStore->pHash, pIter);
return -1;
}
int32_t totLen = INT_BYTES + bodyLen;
void* buf = taosMemoryCalloc(1, totLen);
void* abuf = POINTER_SHIFT(buf, INT_BYTES);
*(int32_t*)buf = htonl(bodyLen);
SEncoder encoder;
tEncoderInit(&encoder, abuf, bodyLen);
tEncodeSTqOffset(&encoder, pOffset);
// write file
int64_t writeLen;
if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
taosHashCancelIterate(pStore->pHash, pIter);
taosMemoryFree(buf);
return -1;
}
taosMemoryFree(buf);
}
// close and rename file
taosCloseFile(&pFile);
pStore->needCommit = 0;
return 0;
taosMemoryFree(pMemBuf);
return code;
}

168
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -1,168 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "meta.h"
#include "tdbInt.h"
#include "tq.h"
// STqOffsetReader ========================================
struct STqOffsetReader {
STQ* pTq;
int64_t sver;
int64_t ever;
int8_t readEnd;
};
int32_t tqOffsetReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetReader** ppReader) {
STqOffsetReader* pReader = NULL;
pReader = taosMemoryCalloc(1, sizeof(STqOffsetReader));
if (pReader == NULL) {
*ppReader = NULL;
return -1;
}
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
tqInfo("vgId:%d, vnode snapshot tq offset reader opened", TD_VID(pTq->pVnode));
*ppReader = pReader;
return 0;
}
int32_t tqOffsetReaderClose(STqOffsetReader** ppReader) {
taosMemoryFree(*ppReader);
*ppReader = NULL;
return 0;
}
int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
if (pReader->readEnd != 0) return 0;
char* fname = tqOffsetBuildFName(pReader->pTq->path, 0);
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
taosMemoryFree(fname);
return 0;
}
int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
taosCloseFile(&pFile);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);
SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr));
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
return terrno;
}
void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr));
int64_t contLen = taosReadFile(pFile, abuf, sz);
if (contLen != sz) {
taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1;
}
buf->size = sz;
buf->type = SNAP_DATA_TQ_OFFSET;
*ppData = (uint8_t*)buf;
pReader->readEnd = 1;
taosCloseFile(&pFile);
return 0;
}
// STqOffseWriter ========================================
struct STqOffsetWriter {
STQ* pTq;
int64_t sver;
int64_t ever;
int32_t tmpFileVer;
char* fname;
};
int32_t tqOffsetWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqOffsetWriter** ppWriter) {
int32_t code = 0;
STqOffsetWriter* pWriter;
pWriter = (STqOffsetWriter*)taosMemoryCalloc(1, sizeof(STqOffsetWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
pWriter->pTq = pTq;
pWriter->sver = sver;
pWriter->ever = ever;
*ppWriter = pWriter;
return code;
_err:
tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
*ppWriter = NULL;
return code;
}
int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) {
STqOffsetWriter* pWriter = *ppWriter;
STQ* pTq = pWriter->pTq;
char* fname = tqOffsetBuildFName(pTq->path, 0);
if (rollback) {
if (taosRemoveFile(pWriter->fname) < 0) {
taosMemoryFree(fname);
return -1;
}
} else {
if (taosRenameFile(pWriter->fname, fname) < 0) {
taosMemoryFree(fname);
return -1;
}
if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) {
taosMemoryFree(fname);
return -1;
}
}
taosMemoryFree(fname);
taosMemoryFree(pWriter->fname);
taosMemoryFree(pWriter);
*ppWriter = NULL;
return 0;
}
int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nData) {
STQ* pTq = pWriter->pTq;
pWriter->tmpFileVer = 1;
pWriter->fname = tqOffsetBuildFName(pTq->path, pWriter->tmpFileVer);
TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE);
SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
int64_t size = pHdr->size;
if (pFile) {
int64_t contLen = taosWriteFile(pFile, pHdr->data, size);
if (contLen != size) {
taosCloseFile(&pFile);
return -1;
}
taosCloseFile(&pFile);
} else {
return -1;
}
return 0;
}

6
source/dnode/vnode/src/tq/tqRead.c
View File

@ -341,10 +341,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
memcpy(data, pBody, len);
SPackedData data1 = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data};
*pItem = (SStreamQueueItem*)streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT);
if (*pItem == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
terrno = code;
code = streamDataSubmitNew(&data1, STREAM_INPUT__DATA_SUBMIT, (SStreamDataSubmit**)pItem);
if (code != 0) {
tqError("%s failed to create data submit for stream since out of memory", id);
return code;
}

62
source/dnode/vnode/src/tq/tqHandleSnapshot.c → source/dnode/vnode/src/tq/tqSnapshot.c
View File

@ -23,9 +23,10 @@ struct STqSnapReader {
int64_t sver;
int64_t ever;
TBC* pCur;
int8_t type;
};
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader) {
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) {
int32_t code = 0;
STqSnapReader* pReader = NULL;
@ -38,9 +39,21 @@ int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** p
pReader->pTq = pTq;
pReader->sver = sver;
pReader->ever = ever;
pReader->type = type;
// impl
code = tdbTbcOpen(pTq->pExecStore, &pReader->pCur, NULL);
TTB *pTb = NULL;
if (type == SNAP_DATA_TQ_CHECKINFO) {
pTb = pTq->pCheckStore;
} else if (type == SNAP_DATA_TQ_HANDLE) {
pTb = pTq->pExecStore;
} else if (type == SNAP_DATA_TQ_OFFSET) {
pTb = pTq->pOffsetStore;
} else {
code = TSDB_CODE_INVALID_MSG;
goto _err;
}
code = tdbTbcOpen(pTb, &pReader->pCur, NULL);
if (code) {
taosMemoryFree(pReader);
goto _err;
@ -91,7 +104,7 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
pHdr->type = SNAP_DATA_TQ_HANDLE;
pHdr->type = pReader->type;
pHdr->size = vLen;
memcpy(pHdr->data, pVal, vLen);
@ -169,7 +182,7 @@ _err:
return code;
}
int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
SDecoder decoder = {0};
@ -180,7 +193,7 @@ int32_t tqSnapWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
code = tDecodeSTqHandle(pDecoder, &handle);
if (code) goto end;
taosWLockLatch(&pTq->lock);
code = tqMetaSaveHandle(pTq, handle.subKey, &handle);
code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
taosWUnLockLatch(&pTq->lock);
end:
@ -189,3 +202,42 @@ end:
tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code);
return code;
}
int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqCheckInfo info = {0};
if(tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){
goto _err;
}
code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
tDeleteSTqCheckInfo(&info);
if (code) goto _err;
return code;
_err:
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}
int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STQ* pTq = pWriter->pTq;
STqOffset info = {0};
if(tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)) != 0){
goto _err;
}
code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
tDeleteSTqOffset(&info);
if (code) goto _err;
return code;
_err:
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
return code;
}

20
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -226,15 +226,15 @@ bool taskReadyForDataFromWal(SStreamTask* pTask) {
}
// not in ready state, do not handle the data from wal
SStreamTaskState* pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__READY) {
tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState->name);
SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__READY) {
tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, pState.name);
return false;
}
// fill-history task has entered into the last phase, no need to anything
if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
ASSERT(pState->state == TASK_STATUS__READY);
ASSERT(pState.state == TASK_STATUS__READY);
// the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer);
@ -324,7 +324,8 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
for (int32_t i = 0; i < numOfTasks; ++i) {
STaskId* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId);
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pStreamMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) {
continue;
}
@ -337,7 +338,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = false;
// seek the stored version and extract data from WAL
int32_t code = setWalReaderStartOffset(pTask, vgId);
code = setWalReaderStartOffset(pTask, vgId);
if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
@ -348,9 +349,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
taosThreadMutexLock(&pTask->lock);
SStreamTaskState* pState = streamTaskGetStatus(pTask);
if (pState->state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState->name);
SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
@ -364,6 +365,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
code = streamTrySchedExec(pTask);
if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask);
taosArrayDestroy(pTaskList);
return -1;
}
}

Some files were not shown because too many files have changed in this diff Show More