xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into enh--refactor-return-code

This commit is contained in:
Hongze Cheng 2024-07-23 15:16:55 +08:00
parent 68eddf62ac 4139dc6663
commit bf37c4f81b
168 changed files with 11363 additions and 7325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -416,7 +416,7 @@ pipeline {
echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
} }
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 150, unit: 'MINUTES'){ timeout(time: 200, unit: 'MINUTES'){
pre_test() pre_test()
script { script {
sh ''' sh '''

2
include/common/rsync.h
View File

@ -12,7 +12,7 @@ extern "C" {
#include "tarray.h" #include "tarray.h"
void stopRsync(); void stopRsync();
void startRsync(); int32_t startRsync();
int32_t uploadByRsync(const char* id, const char* path); int32_t uploadByRsync(const char* id, const char* path);
int32_t downloadRsync(const char* id, const char* path); int32_t downloadRsync(const char* id, const char* path);
int32_t deleteRsync(const char* id); int32_t deleteRsync(const char* id);

2
include/common/tdatablock.h
View File

@ -247,7 +247,7 @@ int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockI
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize);
int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n); int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n);
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n); void blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc); int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);

4
include/common/ttime.h
View File

@ -86,11 +86,11 @@ void deltaToUtcInitOnce();
char getPrecisionUnit(int32_t precision); char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit); int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
struct STm { struct STm {
struct tm tm; struct tm tm;

2
include/libs/function/functionMgt.h
View File

@ -138,6 +138,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_CACHE_LAST_ROW, FUNCTION_TYPE_CACHE_LAST_ROW,
FUNCTION_TYPE_CACHE_LAST, FUNCTION_TYPE_CACHE_LAST,
FUNCTION_TYPE_TABLE_COUNT, FUNCTION_TYPE_TABLE_COUNT,
FUNCTION_TYPE_GROUP_CONST_VALUE,
// distributed splitting functions // distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
@ -256,6 +257,7 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId); bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId); bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId); bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes); void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList);

2
include/libs/qcom/query.h
View File

@ -335,7 +335,7 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam
void destroyQueryExecRes(SExecResult* pRes); void destroyQueryExecRes(SExecResult* pRes);
int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len);
char* parseTagDatatoJson(void* p); void parseTagDatatoJson(void* p, char** jsonStr);
int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst);
void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType);
int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst);

11
include/libs/stream/tstream.h
View File

@ -686,7 +686,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
// checkpoint related // checkpoint related
int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId); void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId);
int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId); int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId);
int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId); int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId);
bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId); bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId);
@ -770,9 +770,9 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask); int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer // timer
tmr_h streamTimerGetInstance(); int32_t streamTimerGetInstance(tmr_h* pTmr);
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg); const char* pMsg);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
@ -809,6 +809,9 @@ void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus #ifdef __cplusplus
} }

1
include/os/osString.h
View File

@ -61,6 +61,7 @@ typedef enum { M2C = 0, C2M } ConvType;
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) #define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src))
char *tstrdup(const char *src); char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosUcs4len(TdUcs4 *ucs4);

3
include/util/taoserror.h
View File

@ -53,7 +53,7 @@ int32_t taosGetErrSize();
#define terrln (*taosGetErrln()) #define terrln (*taosGetErrln())
#define SET_ERROR_MSG(MSG, ...) \ #define SET_ERROR_MSG(MSG, ...) \
snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__) (void)snprintf(terrMsg, ERR_MSG_LEN, MSG, ##__VA_ARGS__)
#define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_SUCCESS 0
#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error
@ -556,6 +556,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B) #define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER TAOS_DEF_ERROR_CODE(0, 0x061B)
#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C) #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR TAOS_DEF_ERROR_CODE(0, 0x061C)
#define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D) #define TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE TAOS_DEF_ERROR_CODE(0, 0x061D)
#define TSDB_CODE_TDB_INCONSISTENT_DB_ID TAOS_DEF_ERROR_CODE(0, 0x061E)
// query // query
#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700)

4
include/util/tutil.h
View File

@ -135,6 +135,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \ } \
} while (0) } while (0)
#define QUERY_CHECK_CODE TSDB_CHECK_CODE
#define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \ #define TSDB_CHECK_NULL(ptr, CODE, LINO, LABEL, ERRNO) \
if ((ptr) == NULL) { \ if ((ptr) == NULL) { \
(CODE) = (ERRNO); \ (CODE) = (ERRNO); \
@ -142,6 +144,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
goto LABEL; \ goto LABEL; \
} }
#define QUERY_CHECK_NULL TSDB_CHECK_NULL
#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL) #define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)

7
packaging/deb/DEBIAN/prerm
View File

@ -20,6 +20,7 @@ if [ -f ${insmetaPath}/preun.sh ]; then
else else
bin_link_dir="/usr/bin" bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib" lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include" inc_link_dir="/usr/include"
data_link_dir="/usr/local/taos/data" data_link_dir="/usr/local/taos/data"
@ -40,9 +41,11 @@ else
${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || : ${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || :
[ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${inc_link_dir}/taosws.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || :
[ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
${csudo}rm -f ${lib_link_dir}/libtaosws.so || :
${csudo}rm -f ${lib64_link_dir}/libtaosws.so || :
${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${log_link_dir} || :
${csudo}rm -f ${data_link_dir} || : ${csudo}rm -f ${data_link_dir} || :

10
packaging/rpm/tdengine.spec
View File

@ -223,6 +223,7 @@ if [ $1 -eq 0 ];then
else else
bin_link_dir="/usr/bin" bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib" lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include" inc_link_dir="/usr/include"
data_link_dir="/usr/local/taos/data" data_link_dir="/usr/local/taos/data"
@ -235,13 +236,18 @@ if [ $1 -eq 0 ];then
${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/taosdump || :
${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${cfg_link_dir}/* || :
${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taos.h || :
${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || :
${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || : ${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${inc_link_dir}/taows.h || :
${csudo}rm -f ${lib_link_dir}/libtaos.so || :
${csudo}rm -f ${lib_link_dir}/libtaosws.so || :
${csudo}rm -f ${lib64_link_dir}/libtaosws.so || :
${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${log_link_dir} || :
${csudo}rm -f ${data_link_dir} || : ${csudo}rm -f ${data_link_dir} || :

5
packaging/tools/remove.sh
View File

@ -12,6 +12,8 @@ RED='\033[0;31m'
GREEN='\033[1;32m' GREEN='\033[1;32m'
NC='\033[0m' NC='\033[0m'
PREFIX="taos"
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
installDir="/usr/local/taos" installDir="/usr/local/taos"
bin_link_dir="/usr/bin" bin_link_dir="/usr/bin"
@ -32,7 +34,6 @@ else
inc_link_dir="/usr/local/include" inc_link_dir="/usr/local/include"
fi fi
PREFIX="taos"
serverName="${PREFIX}d" serverName="${PREFIX}d"
clientName="${PREFIX}" clientName="${PREFIX}"
uninstallScript="rm${PREFIX}" uninstallScript="rm${PREFIX}"
@ -46,8 +47,6 @@ explorerName="${PREFIX}-explorer"
tarbitratorName="tarbitratord" tarbitratorName="tarbitratord"
productName="TDengine" productName="TDengine"
installDir="/usr/local/${PREFIX}"
#install main path #install main path
install_main_dir=${installDir} install_main_dir=${installDir}
data_link_dir=${installDir}/data data_link_dir=${installDir}/data

12
packaging/tools/remove_client.sh
View File

@ -63,10 +63,13 @@ function clean_bin() {
} }
function clean_lib() { function clean_lib() {
# Remove link # Remove link
${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || : [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
${csudo}rm -f ${lib64_link_dir}/libtaos.* || :
[ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || :
#${csudo}rm -rf ${v15_java_app_dir} || :
} }
function clean_header() { function clean_header() {
@ -76,6 +79,7 @@ function clean_header() {
${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || :
${csudo}rm -f ${inc_link_dir}/tdef.h || : ${csudo}rm -f ${inc_link_dir}/tdef.h || :
${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || :
${csudo}rm -f ${inc_link_dir}/taosws.h || :
} }
function clean_config() { function clean_config() {

13
source/client/src/clientEnv.c
View File

@ -35,6 +35,7 @@
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "tversion.h" #include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -875,6 +876,12 @@ void taos_init_imp(void) {
} }
rpcInit(); rpcInit();
if (InitRegexCache() != 0) {
tscInitRes = -1;
tscError("failed to init regex cache");
return;
}
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
catalogInit(&cfg); catalogInit(&cfg);
@ -885,7 +892,11 @@ void taos_init_imp(void) {
taosSetCoreDump(true); taosSetCoreDump(true);
#endif #endif
initTaskQueue(); if (initTaskQueue() != 0){
tscInitRes = -1;
tscError("failed to init task queue");
return;
}
fmFuncMgtInit(); fmFuncMgtInit();
nodesInitAllocatorSet(); nodesInitAllocatorSet();

3
source/client/src/clientImpl.c
View File

@ -2007,7 +2007,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
varDataSetLen(dst, strlen(varDataVal(dst))); varDataSetLen(dst, strlen(varDataVal(dst)));
} else if (tTagIsJson(data)) { } else if (tTagIsJson(data)) {
char* jsonString = parseTagDatatoJson(data); char* jsonString = NULL;
parseTagDatatoJson(data, &jsonString);
STR_TO_VARSTR(dst, jsonString); STR_TO_VARSTR(dst, jsonString);
taosMemoryFree(jsonString); taosMemoryFree(jsonString);
} else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value"

2
source/client/src/clientMain.c
View File

@ -28,6 +28,7 @@
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -78,6 +79,7 @@ void taos_cleanup(void) {
clientConnRefPool = -1; clientConnRefPool = -1;
taosCloseRef(id); taosCloseRef(id);
DestroyRegexCache();
rpcCleanup(); rpcCleanup();
tscDebug("rpc cleanup"); tscDebug("rpc cleanup");

1222
source/client/src/clientRawBlockWrite.c
View File

File diff suppressed because it is too large Load Diff

20
source/client/test/CMakeLists.txt
View File

@ -23,11 +23,11 @@ TARGET_LINK_LIBRARIES(
PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom geometry
) )
ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp) #ADD_EXECUTABLE(clientMonitorTest clientMonitorTests.cpp)
TARGET_LINK_LIBRARIES( #TARGET_LINK_LIBRARIES(
clientMonitorTest # clientMonitorTest
PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor # PUBLIC os util common transport monitor parser catalog scheduler function gtest taos_static qcom executor
) #)
TARGET_INCLUDE_DIRECTORIES( TARGET_INCLUDE_DIRECTORIES(
clientTest clientTest
@ -47,11 +47,11 @@ TARGET_INCLUDE_DIRECTORIES(
PRIVATE "${TD_SOURCE_DIR}/source/client/inc" PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
) )
TARGET_INCLUDE_DIRECTORIES( #TARGET_INCLUDE_DIRECTORIES(
clientMonitorTest # clientMonitorTest
PUBLIC "${TD_SOURCE_DIR}/include/client/" # PUBLIC "${TD_SOURCE_DIR}/include/client/"
PRIVATE "${TD_SOURCE_DIR}/source/client/inc" # PRIVATE "${TD_SOURCE_DIR}/source/client/inc"
) #)
add_test( add_test(
NAME smlTest NAME smlTest

26
source/common/src/rsync.c
View File

@ -54,10 +54,11 @@ static void changeDirFromWindowsToLinux(char* from, char* to) {
#endif #endif
static int32_t generateConfigFile(char* confDir) { static int32_t generateConfigFile(char* confDir) {
int32_t code = 0;
TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -92,7 +93,8 @@ static int32_t generateConfigFile(char* confDir) {
if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) { if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) {
uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; code = TAOS_SYSTEM_ERROR(errno);
return code;
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
@ -129,11 +131,13 @@ void stopRsync() {
taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation. taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation.
} }
void startRsync() { int32_t startRsync() {
int32_t code = 0;
if (taosMulMkDir(tsCheckpointBackupDir) != 0) { if (taosMulMkDir(tsCheckpointBackupDir) != 0) {
uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir, uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir,
ERRNO_ERR_DATA); ERRNO_ERR_DATA);
return; code = TAOS_SYSTEM_ERROR(errno);
return code;
} }
removeEmptyDir(); removeEmptyDir();
@ -141,9 +145,9 @@ void startRsync() {
char confDir[PATH_MAX] = {0}; char confDir[PATH_MAX] = {0};
snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir); snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
int32_t code = generateConfigFile(confDir); code = generateConfigFile(confDir);
if (code != 0) { if (code != 0) {
return; return code;
} }
char cmd[PATH_MAX] = {0}; char cmd[PATH_MAX] = {0};
@ -152,9 +156,11 @@ void startRsync() {
code = system(cmd); code = system(cmd);
if (code != 0) { if (code != 0) {
uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA); uError("[rsync] cmd:%s start server failed, code:%d," ERRNO_ERR_FORMAT, cmd, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
uInfo("[rsync] cmd:%s start server successful", cmd); uInfo("[rsync] cmd:%s start server successful", cmd);
} }
return code;
} }
int32_t uploadByRsync(const char* id, const char* path) { int32_t uploadByRsync(const char* id, const char* path) {
@ -199,6 +205,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) { if (code != 0) {
uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA); tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
int64_t el = (taosGetTimestampMs() - st); int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -242,6 +249,7 @@ int32_t uploadByRsync(const char* id, const char* path) {
if (code != 0) { if (code != 0) {
uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path,
tsSnodeAddress, code, ERRNO_ERR_DATA); tsSnodeAddress, code, ERRNO_ERR_DATA);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
int64_t el = (taosGetTimestampMs() - st); int64_t el = (taosGetTimestampMs() - st);
uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path,
@ -283,13 +291,13 @@ int32_t downloadRsync(const char* id, const char* path) {
uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id, uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id,
path, times, code, ERRNO_ERR_DATA); path, times, code, ERRNO_ERR_DATA);
taosSsleep(1); taosSsleep(1);
code = TAOS_SYSTEM_ERROR(errno);
} else { } else {
int32_t el = taosGetTimestampMs() - st; int32_t el = taosGetTimestampMs() - st;
uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el); uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el);
break; break;
} }
} }
return code; return code;
} }
@ -298,7 +306,7 @@ int32_t deleteRsync(const char* id) {
int32_t code = taosMkDir(tmp); int32_t code = taosMkDir(tmp);
if (code != 0) { if (code != 0) {
uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
char command[PATH_MAX] = {0}; char command[PATH_MAX] = {0};
@ -310,7 +318,7 @@ int32_t deleteRsync(const char* id) {
taosRemoveDir(tmp); taosRemoveDir(tmp);
if (code != 0) { if (code != 0) {
uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
return -1; return TAOS_SYSTEM_ERROR(errno);
} }
uDebug("[rsync] delete data:%s successful", id); uDebug("[rsync] delete data:%s successful", id);

10
source/common/src/tdatablock.c
View File

@ -1990,14 +1990,14 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
} }
} }
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) { void blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
if (n == 0) { if (n == 0) {
blockDataEmpty(pBlock); blockDataEmpty(pBlock);
return TSDB_CODE_SUCCESS; return ;
} }
if (pBlock->info.rows <= n) { if (pBlock->info.rows <= n) {
return TSDB_CODE_SUCCESS; return ;
} else { } else {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
@ -2007,7 +2007,7 @@ int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
pBlock->info.rows = n; pBlock->info.rows = n;
} }
return TSDB_CODE_SUCCESS; return ;
} }
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
@ -2491,7 +2491,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
int32_t code = buildChildTableName(&rname); int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
taosArrayDestroy(tags); taosArrayDestroy(tags);

58
source/common/src/tglobal.c
View File

@ -1370,58 +1370,68 @@ static int32_t taosCheckGlobalCfg() {
return 0; return 0;
} }
static int32_t cfgInitWrapper(SConfig **pCfg) {
if (*pCfg == NULL) {
*pCfg = cfgInit();
if (*pCfg == NULL) {
return terrno;
}
}
return 0;
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) { bool tsc) {
if (tsCfg != NULL) return 0; if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
int32_t code = cfgInitWrapper(&tsCfg);
if (tsc) { if (tsc) {
if (taosAddClientCfg(tsCfg) != 0) return -1; if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if (taosAddClientLogCfg(tsCfg) != 0) return -1; if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
} else { } else {
if (taosAddClientCfg(tsCfg) != 0) return -1; if ((code = taosAddClientCfg(tsCfg)) != 0) return code;
if (taosAddServerCfg(tsCfg) != 0) return -1; if ((code = taosAddServerCfg(tsCfg)) != 0) return code;
if (taosAddClientLogCfg(tsCfg) != 0) return -1; if ((code = taosAddClientLogCfg(tsCfg)) != 0) return code;
if (taosAddServerLogCfg(tsCfg) != 0) return -1; if ((code = taosAddServerLogCfg(tsCfg)) != 0) return code;
} }
taosAddSystemCfg(tsCfg); code = taosAddSystemCfg(tsCfg);
if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
uError("failed to load cfg since %s", terrstr()); uError("failed to load cfg since %s", tstrerror(code));
cfgCleanup(tsCfg); cfgCleanup(tsCfg);
tsCfg = NULL; tsCfg = NULL;
return -1; return code;
} }
if (cfgLoadFromArray(tsCfg, pArgs) != 0) { if ((code = cfgLoadFromArray(tsCfg, pArgs)) != 0) {
uError("failed to load cfg from array since %s", terrstr()); uError("failed to load cfg from array since %s", tstrerror(code));
cfgCleanup(tsCfg); cfgCleanup(tsCfg);
tsCfg = NULL; tsCfg = NULL;
return -1; return code;
} }
if (tsc) { if (tsc) {
if (taosSetClientCfg(tsCfg)) return -1; if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
} else { } else {
if (taosSetClientCfg(tsCfg)) return -1; if ((code = taosSetClientCfg(tsCfg)) != 0) return code;
if (taosUpdateServerCfg(tsCfg)) return -1; if ((code = taosUpdateServerCfg(tsCfg)) != 0) return code;
if (taosSetServerCfg(tsCfg)) return -1; if ((code = taosSetServerCfg(tsCfg)) != 0) return code;
if (taosSetReleaseCfg(tsCfg)) return -1; if ((code = taosSetReleaseCfg(tsCfg)) != 0) return code;
if (taosSetTfsCfg(tsCfg) != 0) return -1; if ((code = taosSetTfsCfg(tsCfg)) != 0) return code;
if (taosSetS3Cfg(tsCfg) != 0) return -1; if ((code = taosSetS3Cfg(tsCfg)) != 0) return code;
} }
taosSetSystemCfg(tsCfg); taosSetSystemCfg(tsCfg);
if (taosSetFileHandlesLimit() != 0) return -1; if ((code = taosSetFileHandlesLimit()) != 0) return code;
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
cfgDumpCfg(tsCfg, tsc, false); cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) { if ((code = taosCheckGlobalCfg()) != 0) {
return -1; return code;
} }
return 0; return 0;

1
source/common/src/tmisce.c
View File

@ -150,7 +150,6 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
cap -= nwrite; cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) { for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) { if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else { } else {

2
source/common/src/tmsg.c
View File

@ -10828,7 +10828,9 @@ int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) {
if (tDecodeI32(pDecoder, &size) < 0) return -1; if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) { if (size > 0) {
pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES); pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES);
if (!pRsp->batchMetaReq) return -1;
pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t)); pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t));
if (!pRsp->batchMetaLen) return -1;
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
void *pCreate = NULL; void *pCreate = NULL;
uint64_t len = 0; uint64_t len = 0;

225
source/common/src/ttime.c
View File

@ -35,7 +35,7 @@ void deltaToUtcInitOnce() {
// printf("====delta:%lld\n\n", seconds); // printf("====delta:%lld\n\n", seconds);
} }
static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction);
static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
@ -95,7 +95,9 @@ char* forwardToTimeStringEnd(char* str) {
return &str[i]; return &str[i];
} }
int64_t parseFraction(char* str, char** end, int32_t timePrec) { int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t i = 0; int32_t i = 0;
int64_t fraction = 0; int64_t fraction = 0;
@ -112,7 +114,7 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
int32_t totalLen = i; int32_t totalLen = i;
if (totalLen <= 0) { if (totalLen <= 0) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
/* parse the fraction */ /* parse the fraction */
@ -134,21 +136,24 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
} }
times = NANO_SEC_FRACTION_LEN - i; times = NANO_SEC_FRACTION_LEN - i;
} else { } else {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
fraction = strnatoi(str, i) * factor[times]; fraction = strnatoi(str, i) * factor[times];
*end = str + totalLen; *end = str + totalLen;
*pFraction = fraction;
return fraction; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t parseTimezone(char* str, int64_t* tzOffset) { int32_t parseTimezone(char* str, int64_t* tzOffset) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t hour = 0; int64_t hour = 0;
int32_t i = 0; int32_t i = 0;
if (str[i] != '+' && str[i] != '-') { if (str[i] != '+' && str[i] != '-') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
i++; i++;
@ -160,7 +165,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
continue; continue;
} }
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
char* sep = strchr(&str[i], ':'); char* sep = strchr(&str[i], ':');
@ -175,18 +180,18 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
} }
if (hour > 12 || hour < 0) { if (hour > 12 || hour < 0) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
// return error if there're illegal charaters after min(2 Digits) // return error if there're illegal charaters after min(2 Digits)
char* minStr = &str[i]; char* minStr = &str[i];
if (minStr[1] != '\0' && minStr[2] != '\0') { if (minStr[1] != '\0' && minStr[2] != '\0') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
int64_t minute = strnatoi(&str[i], 2); int64_t minute = strnatoi(&str[i], 2);
if (minute > 59 || (hour == 12 && minute > 0)) { if (minute > 59 || (hour == 12 && minute > 0)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
if (str[0] == '+') { if (str[0] == '+') {
@ -195,13 +200,13 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
*tzOffset = hour * 3600 + minute * 60; *tzOffset = hour * 3600 + minute * 60;
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) { if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) {
*offset = 0; *offset = 0;
return 0; return TSDB_CODE_SUCCESS;
} }
return parseTimezone(tzStr, offset); return parseTimezone(tzStr, offset);
} }
@ -219,6 +224,8 @@ int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
* 2013-04-12T15:52:01.123+0800 * 2013-04-12T15:52:01.123+0800
*/ */
int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t factor = TSDB_TICK_PER_SECOND(timePrec);
int64_t tzOffset = 0; int64_t tzOffset = 0;
@ -234,7 +241,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
} }
if (str == NULL) { if (str == NULL) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
/* mktime will be affected by TZ, set by using taos_options */ /* mktime will be affected by TZ, set by using taos_options */
@ -253,22 +260,18 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor; *time = seconds * factor;
} else if (str[0] == '.') { } else if (str[0] == '.') {
str += 1; str += 1;
if ((fraction = parseFraction(str, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str, &str, timePrec, &fraction));
return -1;
}
*time = seconds * factor + fraction; *time = seconds * factor + fraction;
char seg = str[0]; char seg = str[0];
if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') { if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') { } else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if (seg == '+' || seg == '-') { } else if (seg == '+' || seg == '-') {
// parse the timezone // parse the timezone
if (parseTimezone(str, &tzOffset) == -1) { TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
return -1;
}
*time += tzOffset * factor; *time += tzOffset * factor;
} }
@ -277,16 +280,14 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor + fraction; *time = seconds * factor + fraction;
// parse the timezone // parse the timezone
if (parseTimezone(str, &tzOffset) == -1) { TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
return -1;
}
*time += tzOffset * factor; *time += tzOffset * factor;
} else { } else {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static FORCE_INLINE bool validateTm(struct tm* pTm) { static FORCE_INLINE bool validateTm(struct tm* pTm) {
@ -314,6 +315,8 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) {
} }
int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0; *utime = 0;
struct tm tm = {0}; struct tm tm = {0};
@ -330,7 +333,7 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
// if parse failed, try "%Y-%m-%d" format // if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm); str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
@ -347,16 +350,16 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
if (*str == '.') { if (*str == '.') {
/* parse the second fraction part */ /* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
return -1;
}
} }
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0; *utime = 0;
struct tm tm = {0}; struct tm tm = {0};
tm.tm_isdst = -1; tm.tm_isdst = -1;
@ -374,7 +377,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
// if parse failed, try "%Y-%m-%d" format // if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm); str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
@ -384,13 +387,11 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
int64_t fraction = 0; int64_t fraction = 0;
if (*str == '.') { if (*str == '.') {
/* parse the second fraction part */ /* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
return -1;
}
} }
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
char getPrecisionUnit(int32_t precision) { char getPrecisionUnit(int32_t precision) {
@ -482,10 +483,12 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre
// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double =
// 1626006833631000064 // 1626006833631000064
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO &&
fromPrecision != TSDB_TIME_PRECISION_NANO) { fromPrecision != TSDB_TIME_PRECISION_NANO) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
@ -541,15 +544,23 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
time *= factors[fromPrecision]; time *= factors[fromPrecision];
break; break;
default: { default: {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
if (tmp >= (double)INT64_MAX) return INT64_MAX; if (tmp >= (double)INT64_MAX) {
if (tmp <= (double)INT64_MIN) return INT64_MIN; *pRes = INT64_MAX;
return time; } else if (tmp <= (double)INT64_MIN) {
*pRes = INT64_MIN;
} else {
*pRes = time;
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) { int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t charLen = varDataLen(inputData); int32_t charLen = varDataLen(inputData);
char* newColData; char* newColData;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
@ -558,7 +569,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return TSDB_CODE_INVALID_TIMESTAMP; TAOS_RETURN(TSDB_CODE_INVALID_TIMESTAMP);
} }
taosMemoryFree(newColData); taosMemoryFree(newColData);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
@ -566,50 +577,52 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData);
if (len < 0) { if (len < 0) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return TSDB_CODE_FAILED; TAOS_RETURN(TSDB_CODE_FAILED);
} }
newColData[len] = 0; newColData[len] = 0;
int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight); int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return ret; TAOS_RETURN(ret);
} }
taosMemoryFree(newColData); taosMemoryFree(newColData);
} else { } else {
return TSDB_CODE_FAILED; TAOS_RETURN(TSDB_CODE_FAILED);
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
switch (unit) { switch (unit) {
case 's': case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) { if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'm': case 'm':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'h': case 'h':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'd': case 'd':
if (val > INT64_MAX / MILLISECOND_PER_DAY) { if (val > INT64_MAX / MILLISECOND_PER_DAY) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'w': case 'w':
if (val > INT64_MAX / MILLISECOND_PER_WEEK) { if (val > INT64_MAX / MILLISECOND_PER_WEEK) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
@ -623,10 +636,10 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
(*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision); (*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision);
break; break;
default: { default: {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
/* /*
@ -645,47 +658,50 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
*/ */
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit,
int32_t timePrecision) { int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0; errno = 0;
char* endPtr = NULL; char* endPtr = NULL;
/* get the basic numeric value */ /* get the basic numeric value */
int64_t timestamp = taosStr2Int64(token, &endPtr, 10); int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
if ((timestamp == 0 && token[0] != '0') || errno != 0) { if ((timestamp == 0 && token[0] != '0') || errno != 0) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
/* natual month/year are not allowed in absolute duration */ /* natual month/year are not allowed in absolute duration */
*unit = token[tokenlen - 1]; *unit = token[tokenlen - 1];
if (*unit == 'n' || *unit == 'y') { if (*unit == 'n' || *unit == 'y') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
return getDuration(timestamp, *unit, duration, timePrecision); return getDuration(timestamp, *unit, duration, timePrecision);
} }
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) { int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision,
bool negativeAllow) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0; errno = 0;
/* get the basic numeric value */ /* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10); *duration = taosStr2Int64(token, NULL, 10);
if ((*duration < 0 && !negativeAllow) || errno != 0) { if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
*unit = token[tokenLen - 1]; *unit = token[tokenLen - 1];
if (*unit == 'n' || *unit == 'y') { if (*unit == 'n' || *unit == 'y') {
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
if(isdigit(*unit)) { if (isdigit(*unit)) {
*unit = getPrecisionUnit(timePrecision); *unit = getPrecisionUnit(timePrecision);
} }
return getDuration(*duration, *unit, duration, timePrecision); return getDuration(*duration, *unit, duration, timePrecision);
} }
static bool taosIsLeapYear(int32_t year) { static bool taosIsLeapYear(int32_t year) { return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); }
return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0));
}
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
if (duration == 0) { if (duration == 0) {
@ -900,7 +916,8 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
// used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point // used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point
int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) {
if (pInterval->offset > 0) { if (pInterval->offset > 0) {
int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t slideStart =
taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision);
int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision);
return result; return result;
@ -960,7 +977,9 @@ const char* fmtts(int64_t ts) {
return buf; return buf;
} }
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
int32_t code = TSDB_CODE_SUCCESS;
char ts[40] = {0}; char ts[40] = {0};
struct tm ptm; struct tm ptm;
@ -996,17 +1015,18 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
default: default:
fractionLen = 0; fractionLen = 0;
return; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
if (taosLocalTime(&quot, &ptm, buf) == NULL) { if (taosLocalTime(&quot, &ptm, buf) == NULL) {
return; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
length += snprintf(ts + length, fractionLen, format, mod); length += snprintf(ts + length, fractionLen, format, mod);
length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
tstrncpy(buf, ts, bufLen); tstrncpy(buf, ts, bufLen);
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) {
@ -1055,7 +1075,7 @@ typedef enum {
TSFKW_Mon, TSFKW_Mon,
TSFKW_MS, TSFKW_MS,
TSFKW_NS, TSFKW_NS,
//TSFKW_OF, // TSFKW_OF,
TSFKW_PM, TSFKW_PM,
TSFKW_P_M, TSFKW_P_M,
TSFKW_SS, TSFKW_SS,
@ -1076,7 +1096,7 @@ typedef enum {
TSFKW_day, TSFKW_day,
TSFKW_ddd, TSFKW_ddd,
TSFKW_dd, TSFKW_dd,
TSFKW_dy, // mon, tue TSFKW_dy, // mon, tue
TSFKW_d, TSFKW_d,
TSFKW_hh24, TSFKW_hh24,
TSFKW_hh12, TSFKW_hh12,
@ -1268,7 +1288,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
} }
if (*formatStr == '\\' && *(formatStr + 1)) { if (*formatStr == '\\' && *(formatStr + 1)) {
formatStr++; formatStr++;
last = NULL; // stop expanding last format, create new format last = NULL; // stop expanding last format, create new format
} }
if (last) { if (last) {
// expand // expand
@ -1289,7 +1309,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
// for other strings // for other strings
if (*formatStr == '\\' && *(formatStr + 1)) { if (*formatStr == '\\' && *(formatStr + 1)) {
formatStr++; formatStr++;
lastOtherFormat = NULL; // stop expanding lastOtherFormat = NULL; // stop expanding
} else { } else {
if (lastOtherFormat && !isSeperatorChar(*formatStr)) { if (lastOtherFormat && !isSeperatorChar(*formatStr)) {
// expanding // expanding
@ -1304,8 +1324,8 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
formatStr++; formatStr++;
} else { } else {
TSFormatNode format = { TSFormatNode format = {
.type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR,
.key = NULL}; .key = NULL};
format.c = formatStr; format.c = formatStr;
format.len = 1; format.len = 1;
taosArrayPush(formats, &format); taosArrayPush(formats, &format);
@ -1318,7 +1338,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
} }
static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
int32_t size = taosArrayGetSize(formats); int32_t size = taosArrayGetSize(formats);
const char* start = s; const char* start = s;
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
TSFormatNode* format = taosArrayGet(formats, i); TSFormatNode* format = taosArrayGet(formats, i);
@ -1923,8 +1943,8 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr
return tm2char(*formats, &tm, out, outLen); return tm2char(*formats, &tm, out, outLen);
} }
int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision,
int32_t errMsgLen) { char* errMsg, int32_t errMsgLen) {
const char* sErrPos; const char* sErrPos;
int32_t fErrIdx; int32_t fErrIdx;
if (!*formats) { if (!*formats) {
@ -1977,26 +1997,27 @@ static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1,
#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] #define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97]
static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ static int64_t UNIT_MATRIX[10][11] = {/* ns, us, ms, s, min, h, d, w, month, y*/
/*ns*/ { 1, 1000, 0}, /*ns*/ {1, 1000, 0},
/*us*/ {1000, 1, 1000, 0}, /*us*/ {1000, 1, 1000, 0},
/*ms*/ { 0, 1000, 1, 1000, 0}, /*ms*/ {0, 1000, 1, 1000, 0},
/*s*/ { 0, 0, 1000, 1, 60, 0}, /*s*/ {0, 0, 1000, 1, 60, 0},
/*min*/ { 0, 0, 0, 60, 1, 60, 0}, /*min*/ {0, 0, 0, 60, 1, 60, 0},
/*h*/ { 0, 0, 0, 0, 60, 1, 1, 0}, /*h*/ {0, 0, 0, 0, 60, 1, 1, 0},
/*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, /*d*/ {0, 0, 0, 0, 0, 24, 1, 7, 1, 0},
/*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, /*w*/ {0, 0, 0, 0, 0, 0, 7, 1, -1, 0},
/*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, /*mon*/ {0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0},
/*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; /*y*/ {0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}};
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) { if (baseIdx == idx) {
if (interval < baseInterval) return false; if (interval < baseInterval) return false;
if (checkEq && interval == baseInterval) return false; if (checkEq && interval == baseInterval) return false;
return interval % baseInterval == 0; return interval % baseInterval == 0;
} }
int8_t next = baseIdx + 1; int8_t next = baseIdx + 1;
int64_t val = UNIT_MATRIX[baseIdx][next]; int64_t val = UNIT_MATRIX[baseIdx][next];
while (val != 0 && next <= idx) { while (val != 0 && next <= idx) {
if (val == -1) { if (val == -1) {
@ -2006,7 +2027,7 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
} }
if (val % baseInterval == 0 || baseInterval % val == 0) { if (val % baseInterval == 0 || baseInterval % val == 0) {
int8_t extra = baseInterval >= val ? 0 : 1; int8_t extra = baseInterval >= val ? 0 : 1;
bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1); bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1);
if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) { if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) {
next++; next++;
val = UNIT_MATRIX[baseIdx][next]; val = UNIT_MATRIX[baseIdx][next];
@ -2021,7 +2042,8 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
return false; return false;
} }
static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) { if (baseIdx == idx) {
@ -2030,7 +2052,7 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
return interval % baseInterval == 0; return interval % baseInterval == 0;
} }
int8_t next = baseIdx - 1; int8_t next = baseIdx - 1;
int64_t val = UNIT_MATRIX[baseIdx][next]; int64_t val = UNIT_MATRIX[baseIdx][next];
while (val != 0 && next >= 0) { while (val != 0 && next >= 0) {
return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq); return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq);
@ -2041,18 +2063,27 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
/* /*
* @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
* @param baseInterval, baseUnit, interval/unit of base tsma * @param baseInterval, baseUnit, interval/unit of base tsma
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already
* translated to TICKS of [precision] * been translated to TICKS of [precision]
* @param unit the tsma unit going to create * @param unit the tsma unit going to create
* @param precision the precision of this db * @param precision the precision of this db
* @param checkEq pass true if same interval is not acceptable, false if acceptable. * @param checkEq pass true if same interval is not acceptable, false if acceptable.
* @ret true the tsma can be created, else cannot * @ret true the tsma can be created, else cannot
* */ * */
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) { bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision,
bool checkEq) {
bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit); bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit);
if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit); if (!baseIsCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit, &baseInterval)) {
return false;
}
}
bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit); bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit);
if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit); if (!isCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(interval, precision, unit, &interval)) {
return false;
}
}
bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq; bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq;

77
source/dnode/mgmt/exe/dmMain.c
View File

@ -167,23 +167,23 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow"); printf("config file path overflow");
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(configDir, argv[i], PATH_MAX); tstrncpy(configDir, argv[i], PATH_MAX);
} else { } else {
printf("'-c' requires a parameter, default is %s\n", configDir); printf("'-c' requires a parameter, default is %s\n", configDir);
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow"); printf("apollo url overflow");
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(global.apolloUrl, argv[i], PATH_MAX); tstrncpy(global.apolloUrl, argv[i], PATH_MAX);
} else { } else {
printf("'-a' requires a parameter\n"); printf("'-a' requires a parameter\n");
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-s") == 0) { } else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true; global.dumpSdb = true;
@ -191,31 +191,31 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if (i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow"); printf("env file path overflow");
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(global.envFile, argv[i], PATH_MAX); tstrncpy(global.envFile, argv[i], PATH_MAX);
} else { } else {
printf("'-E' requires a parameter\n"); printf("'-E' requires a parameter\n");
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-k") == 0) { } else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true; global.generateGrant = true;
} else if (strcmp(argv[i], "-y") == 0) { } else if (strcmp(argv[i], "-y") == 0) {
global.generateCode = true; global.generateCode = true;
if(i < argc - 1) { if (i < argc - 1) {
int32_t len = strlen(argv[++i]); int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) { if (len < ENCRYPT_KEY_LEN_MIN) {
printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN); printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1; return TSDB_CODE_INVALID_CFG;
} }
if (len > ENCRYPT_KEY_LEN) { if (len > ENCRYPT_KEY_LEN) {
printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN); printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1; return TSDB_CODE_INVALID_CFG;
} }
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN); tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
} else { } else {
printf("'-y' requires a parameter\n"); printf("'-y' requires a parameter\n");
return -1; return TSDB_CODE_INVALID_CFG;
} }
} else if (strcmp(argv[i], "-C") == 0) { } else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true; global.dumpConfig = true;
@ -310,6 +310,7 @@ static void taosCleanupArgs() {
} }
int main(int argc, char const *argv[]) { int main(int argc, char const *argv[]) {
int32_t code = 0;
#ifdef TD_JEMALLOC_ENABLED #ifdef TD_JEMALLOC_ENABLED
bool jeBackgroundThread = true; bool jeBackgroundThread = true;
mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool)); mallctl("background_thread", NULL, NULL, &jeBackgroundThread, sizeof(bool));
@ -319,10 +320,10 @@ int main(int argc, char const *argv[]) {
return -1; return -1;
} }
if (dmParseArgs(argc, argv) != 0) { if ((code = dmParseArgs(argc, argv)) != 0) {
//printf("failed to start since parse args error\n"); // printf("failed to start since parse args error\n");
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
#ifdef WINDOWS #ifdef WINDOWS
@ -335,6 +336,7 @@ int main(int argc, char const *argv[]) {
return 0; return 0;
} }
int mainWindows(int argc, char **argv) { int mainWindows(int argc, char **argv) {
int32_t code = 0;
#endif #endif
if (global.generateGrant) { if (global.generateGrant) {
@ -357,7 +359,7 @@ int mainWindows(int argc, char **argv) {
#if defined(LINUX) #if defined(LINUX)
if (global.memDbg) { if (global.memDbg) {
int32_t code = taosMemoryDbgInit(); code = taosMemoryDbgInit();
if (code) { if (code) {
printf("failed to init memory dbg, error:%s\n", tstrerror(code)); printf("failed to init memory dbg, error:%s\n", tstrerror(code));
return code; return code;
@ -366,16 +368,18 @@ int mainWindows(int argc, char **argv) {
printf("memory dbg enabled\n"); printf("memory dbg enabled\n");
} }
#endif #endif
if(global.generateCode) { if (global.generateCode) {
bool toLogFile = false; bool toLogFile = false;
if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){ if ((code = taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs)) != 0) {
encryptError("failed to generate encrypt code since dataDir can not be set from cfg file"); encryptError("failed to generate encrypt code since dataDir can not be set from cfg file,reason:%s",
return -1; tstrerror(code));
return code;
}; };
TdFilePtr pFile;
if(dmCheckRunning(tsDataDir) == NULL) { if ((code = dmCheckRunning(tsDataDir, &pFile)) != 0) {
encryptError("failed to generate encrypt code since taosd is running, please stop it first"); encryptError("failed to generate encrypt code since taosd is running, please stop it first, reason:%s",
return -1; tstrerror(code));
return code;
} }
int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile); int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile);
taosCloseLog(); taosCloseLog();
@ -383,30 +387,30 @@ int mainWindows(int argc, char **argv) {
return ret; return ret;
} }
if (dmInitLog() != 0) { if ((code = dmInitLog()) != 0) {
printf("failed to start since init log error\n"); printf("failed to start since init log error\n");
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
dmPrintArgs(argc, argv); dmPrintArgs(argc, argv);
if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { if ((code = taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0)) != 0) {
dError("failed to start since read config error"); dError("failed to start since read config error");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
if (taosConvInit() != 0) { if ((code = taosConvInit()) != 0) {
dError("failed to init conv"); dError("failed to init conv");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
} }
if (global.checkS3) { if (global.checkS3) {
int32_t code = dmCheckS3(); code = dmCheckS3();
taosCleanupCfg(); taosCleanupCfg();
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
@ -435,31 +439,32 @@ int mainWindows(int argc, char **argv) {
osSetProcPath(argc, (char **)argv); osSetProcPath(argc, (char **)argv);
taosCleanupArgs(); taosCleanupArgs();
if(dmGetEncryptKey() != 0){ if ((code = dmGetEncryptKey()) != 0) {
dError("failed to start since failed to get encrypt key"); dError("failed to start since failed to get encrypt key");
taosCloseLog(); taosCloseLog();
taosCleanupArgs(); taosCleanupArgs();
return -1; return code;
}; };
if (dmInit() != 0) { if ((code = dmInit()) != 0) {
if (terrno == TSDB_CODE_NOT_FOUND) { if (code == TSDB_CODE_NOT_FOUND) {
dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support"); dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support");
} else { } else {
dError("failed to init dnode since %s", terrstr()); dError("failed to init dnode since %s", tstrerror(code));
} }
taosCleanupCfg(); taosCleanupCfg();
taosCloseLog(); taosCloseLog();
taosConvDestroy(); taosConvDestroy();
return -1; return code;
} }
dInfo("start to init service"); dInfo("start to init service");
dmSetSignalHandle(); dmSetSignalHandle();
tsDndStart = taosGetTimestampMs(); tsDndStart = taosGetTimestampMs();
tsDndStartOsUptime = taosGetOsUptime(); tsDndStartOsUptime = taosGetOsUptime();
int32_t code = dmRun();
code = dmRun();
dInfo("shutting down the service"); dInfo("shutting down the service");
dmCleanup(); dmCleanup();

127
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -45,7 +45,7 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer}; SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer};
int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req); int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen); void *pHead = rpcMallocCont(contLen);
tSerializeRetrieveIpWhite(pHead, contLen, &req); tSerializeRetrieveIpWhite(pHead, contLen, &req);
SRpcMsg rpcMsg = {.pCont = pHead, SRpcMsg rpcMsg = {.pCont = pHead,
@ -116,7 +116,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite; req.clusterCfg.ttlChangeOnWrite = tsTtlChangeOnWrite;
req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0; req.clusterCfg.enableWhiteList = tsEnableWhiteList ? 1 : 0;
req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat; req.clusterCfg.encryptionKeyStat = tsEncryptionKeyStat;
req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum; req.clusterCfg.encryptionKeyChksum = tsEncryptionKeyChksum;
req.clusterCfg.monitorParas.tsEnableMonitor = tsEnableMonitor; req.clusterCfg.monitorParas.tsEnableMonitor = tsEnableMonitor;
req.clusterCfg.monitorParas.tsMonitorInterval = tsMonitorInterval; req.clusterCfg.monitorParas.tsMonitorInterval = tsMonitorInterval;
req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope; req.clusterCfg.monitorParas.tsSlowLogScope = tsSlowLogScope;
@ -146,7 +146,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.ipWhiteVer = pMgmt->pData->ipWhiteVer; req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
void * pHead = rpcMallocCont(contLen); void *pHead = rpcMallocCont(contLen);
tSerializeSStatusReq(pHead, contLen, &req); tSerializeSStatusReq(pHead, contLen, &req);
tFreeSStatusReq(&req); tFreeSStatusReq(&req);
@ -207,18 +207,26 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SDCfgDnodeReq cfgReq = {0}; SDCfgDnodeReq cfgReq = {0};
if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
SConfig *pCfg = taosGetCfg(); SConfig *pCfg = taosGetCfg();
cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
taosCfgDynamicOptions(pCfg, cfgReq.config, true); code = cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD, true);
return 0; if (code != 0) {
if (strncasecmp(cfgReq.config, "resetlog", strlen("resetlog")) == 0) {
code = 0;
} else {
return code;
}
}
return taosCfgDynamicOptions(pCfg, cfgReq.config, true);
} }
int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
@ -251,7 +259,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK; pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_OK;
pStatus->details[0] = 0; pStatus->details[0] = 0;
SMonMloadInfo minfo = {0}; SMonMloadInfo minfo = {0};
(*pMgmt->getMnodeLoadsFp)(&minfo); (*pMgmt->getMnodeLoadsFp)(&minfo);
if (minfo.isMnode && if (minfo.isMnode &&
(minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) {
@ -276,32 +284,49 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) {
} }
int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
dDebug("server run status req is received"); dDebug("server run status req is received");
SServerStatusRsp statusRsp = {0}; SServerStatusRsp statusRsp = {0};
dmGetServerRunStatus(pMgmt, &statusRsp); dmGetServerRunStatus(pMgmt, &statusRsp);
pMsg->info.rsp = NULL;
pMsg->info.rspLen = 0;
SRpcMsg rspMsg = {.info = pMsg->info}; SRpcMsg rspMsg = {.info = pMsg->info};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) { if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1; // rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
} }
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1; // rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;
// return rspMsg.code;
}
rspLen = tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
if (rspLen < 0) {
return TSDB_CODE_INVALID_MSG;
} }
tSerializeSServerStatusRsp(pRsp, rspLen, &statusRsp);
pMsg->info.rsp = pRsp; pMsg->info.rsp = pRsp;
pMsg->info.rspLen = rspLen; pMsg->info.rspLen = rspLen;
return 0; return 0;
} }
SSDataBlock *dmBuildVariablesBlock(void) { int32_t dmBuildVariablesBlock(SSDataBlock **ppBlock) {
SSDataBlock * pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); int32_t code = 0;
size_t size = 0;
SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
size_t size = 0;
const SSysTableMeta *pMeta = NULL; const SSysTableMeta *pMeta = NULL;
getInfosDbMeta(&pMeta, &size); getInfosDbMeta(&pMeta, &size);
@ -314,52 +339,74 @@ SSDataBlock *dmBuildVariablesBlock(void) {
} }
pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData)); pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData));
if (pBlock->pDataBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
for (int32_t i = 0; i < pMeta[index].colNum; ++i) { for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData = {0}; SColumnInfoData colInfoData = {0};
colInfoData.info.colId = i + 1; colInfoData.info.colId = i + 1;
colInfoData.info.type = pMeta[index].schema[i].type; colInfoData.info.type = pMeta[index].schema[i].type;
colInfoData.info.bytes = pMeta[index].schema[i].bytes; colInfoData.info.bytes = pMeta[index].schema[i].bytes;
taosArrayPush(pBlock->pDataBlock, &colInfoData); if (taosArrayPush(pBlock->pDataBlock, &colInfoData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
pBlock->info.hasVarCol = true; pBlock->info.hasVarCol = true;
_exit:
return pBlock; if (code != 0) {
blockDataDestroy(pBlock);
} else {
*ppBlock = pBlock;
}
return code;
} }
int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
/*int32_t code = */dumpConfToDataBlock(pBlock, 1); int32_t code = dumpConfToDataBlock(pBlock, 1);
if (code != 0) {
return code;
}
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false); if (pColInfo == NULL) {
return TSDB_CODE_OUT_OF_RANGE;
}
return TSDB_CODE_SUCCESS; return colDataSetNItems(pColInfo, 0, (const char *)&dnodeId, pBlock->info.rows, false);
} }
int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t size = 0; int32_t size = 0;
int32_t rowsRead = 0; int32_t rowsRead = 0;
int32_t code = 0;
SRetrieveTableReq retrieveReq = {0}; SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) { if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
#if 0 #if 0
if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) {
terrno = TSDB_CODE_MND_NO_RIGHTS; code = TSDB_CODE_MND_NO_RIGHTS;
return -1; return code;
} }
#endif #endif
if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
SSDataBlock *pBlock = dmBuildVariablesBlock(); SSDataBlock *pBlock = NULL;
if ((code = dmBuildVariablesBlock(&pBlock)) != 0) {
return code;
}
dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); code = dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId);
if (code != 0) {
blockDataDestroy(pBlock);
return code;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols +
@ -367,10 +414,10 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to retrieve data since %s", terrstr()); dError("failed to retrieve data since %s", tstrerror(code));
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; return code;
} }
char *pStart = pRsp->data; char *pStart = pRsp->data;
@ -404,7 +451,9 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SArray *dmGetMsgHandles() { SArray *dmGetMsgHandles() {
int32_t code = -1; int32_t code = -1;
SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle)); SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle));
if (pArray == NULL) goto _OVER; if (pArray == NULL) {
return NULL;
}
// Requests handled by DNODE // Requests handled by DNODE
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
@ -416,7 +465,7 @@ SArray *dmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_ENCRYPT_KEY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// Requests handled by MNODE // Requests handled by MNODE

32
source/dnode/mgmt/mgmt_dnode/src/dmInt.c
View File

@ -18,22 +18,23 @@
#include "libs/function/tudf.h" #include "libs/function/tudf.h"
static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
if (dmStartStatusThread(pMgmt) != 0) { int32_t code = 0;
return -1; if ((code = dmStartStatusThread(pMgmt)) != 0) {
return code;
} }
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (dmStartNotifyThread(pMgmt) != 0) { if ((code = dmStartNotifyThread(pMgmt)) != 0) {
return -1; return code;
} }
#endif #endif
if (dmStartMonitorThread(pMgmt) != 0) { if ((code = dmStartMonitorThread(pMgmt)) != 0) {
return -1; return code;
} }
if (dmStartAuditThread(pMgmt) != 0) { if ((code = dmStartAuditThread(pMgmt)) != 0) {
return -1; return code;
} }
if (dmStartCrashReportThread(pMgmt) != 0) { if ((code = dmStartCrashReportThread(pMgmt)) != 0) {
return -1; return code;
} }
return 0; return 0;
} }
@ -50,10 +51,10 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) {
} }
static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -70,12 +71,11 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp; pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp;
// pMgmt->pData->ipWhiteVer = 0; if ((code = dmStartWorker(pMgmt)) != 0) {
if (dmStartWorker(pMgmt) != 0) { return code;
return -1;
} }
if (udfStartUdfd(pMgmt->pData->dnodeId) != 0) { if ((code = udfStartUdfd(pMgmt->pData->dnodeId)) != 0) {
dError("failed to start udfd"); dError("failed to start udfd");
} }

34
source/dnode/mgmt/mgmt_dnode/src/dmWorker.c
View File

@ -264,12 +264,14 @@ static void *dmCrashReportThreadFp(void *param) {
} }
int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) {
dError("failed to create status thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create status thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -285,12 +287,14 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) {
dError("failed to create notify thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create notify thread since %s", strerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -308,12 +312,14 @@ void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->monitorThread, &thAttr, dmMonitorThreadFp, pMgmt) != 0) {
dError("failed to create monitor thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create monitor thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -322,12 +328,14 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) { int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
dError("failed to create audit thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create audit thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -350,6 +358,7 @@ void dmStopAuditThread(SDnodeMgmt *pMgmt) {
} }
int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) { int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
int32_t code = 0;
if (!tsEnableCrashReport) { if (!tsEnableCrashReport) {
return 0; return 0;
} }
@ -358,8 +367,9 @@ int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->crashReportThread, &thAttr, dmCrashReportThreadFp, pMgmt) != 0) {
dError("failed to create crashReport thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create crashReport thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -431,8 +441,8 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg); code = dmProcessCreateEncryptKeyReq(pMgmt, pMsg);
break; break;
default: default:
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed in mgmt queue", pMsg); dGError("msg:%p, not processed in mgmt queue, reason:%s", pMsg, tstrerror(code));
break; break;
} }

135
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -21,11 +21,11 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
int32_t code = 0; int32_t code = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code); tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code);
if (code < 0) return -1; if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code);
if (code < 0) return 0; if (code < 0) return code;
tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code); tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code);
if (code < 0) return 0; if (code < 0) return code;
SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); SJson *replicas = tjsonGetObjectItem(pJson, "replicas");
if (replicas == NULL) return 0; if (replicas == NULL) return 0;
@ -35,17 +35,17 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonGetArrayItem(replicas, i); SJson *replica = tjsonGetArrayItem(replicas, i);
if (replica == NULL) return -1; if (replica == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
SReplica *pReplica = pOption->replicas + i; SReplica *pReplica = pOption->replicas + i;
tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code); tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code);
if (code < 0) return -1; if (code < 0) return code;
code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn); code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn);
if (code < 0) return -1; if (code < 0) return code;
tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code);
if (code < 0) return -1; if (code < 0) return code;
tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code); tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
if (code < 0) return -1; if (code < 0) return code;
if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) { if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
pOption->numOfReplicas++; pOption->numOfReplicas++;
} }
@ -63,36 +63,41 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char *pData = NULL; char *pData = NULL;
SJson *pJson = NULL; SJson *pJson = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file); dInfo("mnode file:%s not exist, reason:%s", file, tstrerror(TAOS_SYSTEM_ERROR(errno)));
return 0; return 0;
} }
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open mnode file:%s since %s", file, terrstr()); dError("failed to open mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr()); dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
pData = taosMemoryMalloc(size + 1); pData = taosMemoryMalloc(size + 1);
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, pData, size) != size) { if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read mnode file:%s since %s", file, terrstr()); dError("failed to read mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -100,12 +105,11 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
pJson = tjsonParse(pData); pJson = tjsonParse(pData);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (mmDecodeOption(pJson, pOption) < 0) { if ((code = mmDecodeOption(pJson, pOption)) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -118,37 +122,42 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read mnode file:%s since %s", file, terrstr()); dError("failed to read mnode file:%s since %s", file, tstrerror(code));
} }
return code; return code;
} }
static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) {
int32_t code = 0;
if (pOption->deploy && pOption->numOfTotalReplicas > 0) { if (pOption->deploy && pOption->numOfTotalReplicas > 0) {
if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1; if ((code = tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex)) < 0) return code;
SJson *replicas = tjsonCreateArray(); SJson *replicas = tjsonCreateArray();
if (replicas == NULL) return -1; if (replicas == NULL) {
if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1; return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "replicas", replicas)) < 0) return code;
for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) {
SJson *replica = tjsonCreateObject(); SJson *replica = tjsonCreateObject();
if (replica == NULL) return -1; if (replica == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
const SReplica *pReplica = pOption->replicas + i; const SReplica *pReplica = pOption->replicas + i;
if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1; if ((code = tjsonAddDoubleToObject(replica, "id", pReplica->id)) < 0) return code;
if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1; if ((code = tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn)) < 0) return code;
if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1; if ((code = tjsonAddDoubleToObject(replica, "port", pReplica->port)) < 0) return code;
if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1; if ((code = tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i])) < 0) return code;
if (tjsonAddItemToArray(replicas, replica) < 0) return -1; if ((code = tjsonAddItemToArray(replicas, replica)) < 0) return code;
} }
} }
if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1; if ((code = tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex)) < 0) return code;
if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1; if ((code = tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy)) < 0) return code;
return 0; return code;
} }
int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) { int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
@ -158,28 +167,59 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t nBytes = snprintf(file, sizeof(file), "%s%smnode.json.bak", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%smnode.json", path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
// terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) {
if (mmEncodeOption(pJson, pOption) != 0) goto _OVER; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
TAOS_CHECK_GOTO(mmEncodeOption(pJson, pOption), NULL, _OVER);
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosCloseFile(&pFile) < 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0;
dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy); dInfo("succeed to write mnode file:%s, deloyed:%d", realfile, pOption->deploy);
_OVER: _OVER:
@ -188,8 +228,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, tstrerror(code), pOption->deploy);
dError("failed to write mnode file:%s since %s, deloyed:%d", realfile, terrstr(), pOption->deploy);
} }
return code; return code;
} }

37
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -26,11 +26,12 @@ void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) {
} }
int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SDCreateMnodeReq createReq = {0}; SDCreateMnodeReq createReq = {0};
if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
SMnodeOpt option = {.deploy = true, SMnodeOpt option = {.deploy = true,
@ -56,43 +57,45 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
if (option.selfIndex == -1) { if (option.selfIndex == -1) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dGError("failed to create mnode since %s, selfIndex is -1", terrstr()); dGError("failed to create mnode since %s, selfIndex is -1", tstrerror(code));
return -1; return code;
} }
if (mmWriteFile(pInput->path, &option) != 0) { if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", terrstr()); dGError("failed to write mnode file since %s", tstrerror(code));
return -1; return code;
} }
return 0; return 0;
} }
int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SDDropMnodeReq dropReq = {0}; SDDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dGError("failed to drop mnode since %s", terrstr()); dGError("failed to drop mnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
SMnodeOpt option = {.deploy = false}; SMnodeOpt option = {.deploy = false};
if (mmWriteFile(pInput->path, &option) != 0) { if ((code = mmWriteFile(pInput->path, &option)) != 0) {
dGError("failed to write mnode file since %s", terrstr()); dGError("failed to write mnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return 0; return code;
} }
SArray *mmGetMsgHandles() { SArray *mmGetMsgHandles() {

68
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -25,9 +25,10 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) {
} }
static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
int32_t code = 0;
SMnodeOpt option = {0}; SMnodeOpt option = {0};
if (mmReadFile(pInput->path, &option) != 0) { if ((code = mmReadFile(pInput->path, &option)) != 0) {
return -1; return code;
} }
if (!option.deploy) { if (!option.deploy) {
@ -41,7 +42,7 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
dInfo("deploy mnode required. option deploy:%d", option.deploy); dInfo("deploy mnode required. option deploy:%d", option.deploy);
} }
return 0; return code;
} }
static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) { static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInput, SMnodeOpt *pOption) {
@ -73,22 +74,31 @@ static void mmClose(SMnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
static int32_t mndOpenWrapper(const char *path, SMnodeOpt *opt, SMnode **pMnode) {
int32_t code = 0;
*pMnode = mndOpen(path, opt);
if (*pMnode == NULL) {
code = terrno;
}
///*pMnode = pNode;
return code;
}
static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (walInit() != 0) { int32_t code = 0;
dError("failed to init wal since %s", terrstr()); if ((code = walInit()) != 0) {
return -1; dError("failed to init wal since %s", tstrerror(code));
return code;
} }
if (syncInit() != 0) { if ((code = syncInit()) != 0) {
dError("failed to init sync since %s", terrstr()); dError("failed to init sync since %s", tstrerror(code));
return -1; return code;
} }
SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt)); SMnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SMnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -100,10 +110,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
taosThreadRwlockInit(&pMgmt->lock, NULL); taosThreadRwlockInit(&pMgmt->lock, NULL);
SMnodeOpt option = {0}; SMnodeOpt option = {0};
if (mmReadFile(pMgmt->path, &option) != 0) { if ((code = mmReadFile(pMgmt->path, &option)) != 0) {
dError("failed to read file since %s", terrstr()); dError("failed to read file since %s", tstrerror(code));
mmClose(pMgmt); mmClose(pMgmt);
return -1; return code;
} }
if (!option.deploy) { if (!option.deploy) {
@ -115,18 +125,18 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
mmBuildOptionForOpen(pMgmt, &option); mmBuildOptionForOpen(pMgmt, &option);
} }
pMgmt->pMnode = mndOpen(pMgmt->path, &option); code = mndOpenWrapper(pMgmt->path, &option, &pMgmt->pMnode);
if (pMgmt->pMnode == NULL) { if (code != 0) {
dError("failed to open mnode since %s", terrstr()); dError("failed to open mnode since %s", tstrerror(code));
mmClose(pMgmt); mmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("mnode-impl", "initialized"); tmsgReportStartup("mnode-impl", "initialized");
if (mmStartWorker(pMgmt) != 0) { if ((code = mmStartWorker(pMgmt)) != 0) {
dError("failed to start mnode worker since %s", terrstr()); dError("failed to start mnode worker since %s", tstrerror(code));
mmClose(pMgmt); mmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("mnode-worker", "initialized"); tmsgReportStartup("mnode-worker", "initialized");
@ -134,9 +144,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
option.deploy = true; option.deploy = true;
option.numOfReplicas = 0; option.numOfReplicas = 0;
option.numOfTotalReplicas = 0; option.numOfTotalReplicas = 0;
if (mmWriteFile(pMgmt->path, &option) != 0) { if ((code = mmWriteFile(pMgmt->path, &option)) != 0) {
dError("failed to write mnode file since %s", terrstr()); dError("failed to write mnode file since %s", tstrerror(code));
return -1; return code;
} }
} }
@ -160,13 +170,9 @@ static void mmStop(SMnodeMgmt *pMgmt) {
mndStop(pMgmt->pMnode); mndStop(pMgmt->pMnode);
} }
static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { return mndIsCatchUp(pMgmt->pMnode); }
return mndIsCatchUp(pMgmt->pMnode);
}
static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { return mndGetRole(pMgmt->pMnode); }
return mndGetRole(pMgmt->pMnode);
}
SMgmtFunc mmGetMgmtFunc() { SMgmtFunc mmGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0}; SMgmtFunc mgmtFunc = {0};

51
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -179,13 +179,15 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pWorker = &pMgmt->syncRdWorker; pWorker = &pMgmt->syncRdWorker;
break; break;
default: default:
terrno = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
} }
if (pWorker == NULL) return -1; if (pWorker == NULL) return code;
SRpcMsg *pMsg; SRpcMsg *pMsg;
code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg); code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) return code; if (code) return code;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL; pRpc->pCont = NULL;
@ -201,6 +203,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
} }
int32_t mmStartWorker(SMnodeMgmt *pMgmt) { int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg qCfg = { SSingleWorkerCfg qCfg = {
.min = tsNumOfMnodeQueryThreads, .min = tsNumOfMnodeQueryThreads,
.max = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads,
@ -209,9 +212,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.param = pMgmt, .param = pMgmt,
.poolType = QUERY_AUTO_QWORKER_POOL, .poolType = QUERY_AUTO_QWORKER_POOL,
}; };
if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &qCfg)) != 0) {
dError("failed to start mnode-query worker since %s", terrstr()); dError("failed to start mnode-query worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg fCfg = { SSingleWorkerCfg fCfg = {
@ -221,9 +224,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg)) != 0) {
dError("failed to start mnode-fetch worker since %s", terrstr()); dError("failed to start mnode-fetch worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg rCfg = { SSingleWorkerCfg rCfg = {
@ -233,9 +236,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->readWorker, &rCfg)) != 0) {
dError("failed to start mnode-read worker since %s", terrstr()); dError("failed to start mnode-read worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg wCfg = { SSingleWorkerCfg wCfg = {
@ -245,9 +248,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->writeWorker, &wCfg)) != 0) {
dError("failed to start mnode-write worker since %s", terrstr()); dError("failed to start mnode-write worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg sCfg = { SSingleWorkerCfg sCfg = {
@ -257,9 +260,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg, .fp = (FItem)mmProcessSyncMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->syncWorker, &sCfg)) != 0) {
dError("failed to start mnode mnode-sync worker since %s", terrstr()); dError("failed to start mnode mnode-sync worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg scCfg = { SSingleWorkerCfg scCfg = {
@ -269,9 +272,9 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessSyncMsg, .fp = (FItem)mmProcessSyncMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg)) != 0) {
dError("failed to start mnode mnode-sync-rd worker since %s", terrstr()); dError("failed to start mnode mnode-sync-rd worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg arbCfg = { SSingleWorkerCfg arbCfg = {
@ -281,13 +284,13 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
.fp = (FItem)mmProcessRpcMsg, .fp = (FItem)mmProcessRpcMsg,
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->arbWorker, &arbCfg)) != 0) {
dError("failed to start mnode mnode-arb worker since %s", terrstr()); dError("failed to start mnode mnode-arb worker since %s", tstrerror(code));
return -1; return code;
} }
dDebug("mnode workers are initialized"); dDebug("mnode workers are initialized");
return 0; return code;
} }
void mmStopWorker(SMnodeMgmt *pMgmt) { void mmStopWorker(SMnodeMgmt *pMgmt) {

34
source/dnode/mgmt/mgmt_qnode/src/qmHandle.c
View File

@ -30,24 +30,25 @@ void qmGetQnodeLoads(SQnodeMgmt *pMgmt, SQnodeLoad *pInfo) {
} }
int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0}; SDCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to create qnode since %s", terrstr()); dError("failed to create qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
bool deployed = true; bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", terrstr()); dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
@ -55,24 +56,25 @@ int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0}; SDDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop qnode since %s", terrstr()); dError("failed to drop qnode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
bool deployed = false; bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write qnode file since %s", terrstr()); dError("failed to write qnode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);

30
source/dnode/mgmt/mgmt_qnode/src/qmInt.c
View File

@ -33,11 +33,18 @@ static void qmClose(SQnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption);
if (*pQnode == NULL) {
return terrno;
}
return 0;
}
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt)); SQnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SQnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -50,29 +57,30 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SQnodeOpt option = {0}; SQnodeOpt option = {0};
qmInitOption(pMgmt, &option); qmInitOption(pMgmt, &option);
pMgmt->pQnode = qndOpen(&option);
if (pMgmt->pQnode == NULL) { code = qndOpenWrapper(&option, &pMgmt->pQnode);
dError("failed to open qnode since %s", terrstr()); if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt); qmClose(pMgmt);
return -1; return -1;
} }
tmsgReportStartup("qnode-impl", "initialized"); tmsgReportStartup("qnode-impl", "initialized");
if (udfcOpen() != 0) { if ((code = udfcOpen()) != 0) {
dError("qnode can not open udfc"); dError("qnode can not open udfc");
qmClose(pMgmt); qmClose(pMgmt);
return -1; return code;
} }
if (qmStartWorker(pMgmt) != 0) { if ((code = qmStartWorker(pMgmt)) != 0) {
dError("failed to start qnode worker since %s", terrstr()); dError("failed to start qnode worker since %s", tstrerror(code));
qmClose(pMgmt); qmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("qnode-worker", "initialized"); tmsgReportStartup("qnode-worker", "initialized");
pOutput->pMgmt = pMgmt; pOutput->pMgmt = pMgmt;
return 0; return code;
} }
SMgmtFunc qmGetMgmtFunc() { SMgmtFunc qmGetMgmtFunc() {

16
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -102,6 +102,8 @@ int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
} }
int32_t qmStartWorker(SQnodeMgmt *pMgmt) { int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
int32_t code = 0;
SSingleWorkerCfg queryCfg = { SSingleWorkerCfg queryCfg = {
.min = tsNumOfVnodeQueryThreads, .min = tsNumOfVnodeQueryThreads,
.max = tsNumOfVnodeQueryThreads, .max = tsNumOfVnodeQueryThreads,
@ -111,9 +113,9 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.poolType = QUERY_AUTO_QWORKER_POOL, .poolType = QUERY_AUTO_QWORKER_POOL,
}; };
if (tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg)) != 0) {
dError("failed to start qnode-query worker since %s", terrstr()); dError("failed to start qnode-query worker since %s", tstrerror(code));
return -1; return code;
} }
SSingleWorkerCfg fetchCfg = { SSingleWorkerCfg fetchCfg = {
@ -124,13 +126,13 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg) != 0) { if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg)) != 0) {
dError("failed to start qnode-fetch worker since %s", terrstr()); dError("failed to start qnode-fetch worker since %s", tstrerror(code));
return -1; return code;
} }
dDebug("qnode workers are initialized"); dDebug("qnode workers are initialized");
return 0; return code;
} }
void qmStopWorker(SQnodeMgmt *pMgmt) { void qmStopWorker(SQnodeMgmt *pMgmt) {

35
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -19,24 +19,25 @@
void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {} void smGetMonitorInfo(SSnodeMgmt *pMgmt, SMonSmInfo *smInfo) {}
int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0}; SDCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to create snode since %s", terrstr()); dError("failed to create snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
bool deployed = true; bool deployed = true;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", terrstr()); dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
@ -44,24 +45,26 @@ int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1;
return code;
} }
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; code = TSDB_CODE_INVALID_OPTION;
dError("failed to drop snode since %s", terrstr()); dError("failed to drop snode since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
bool deployed = false; bool deployed = false;
if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { if ((code = dmWriteFile(pInput->path, pInput->name, deployed)) != 0) {
dError("failed to write snode file since %s", terrstr()); dError("failed to write snode file since %s", tstrerror(code));
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return -1; return code;
} }
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);

38
source/dnode/mgmt/mgmt_snode/src/smInt.c
View File

@ -33,12 +33,19 @@ static void smClose(SSnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
int32_t sndOpenWrapper(const char *path, SSnodeOpt *pOption, SSnode **pNode) {
*pNode = sndOpen(path, pOption);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt)); SSnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SSnodeMgmt));
if (pMgmt == NULL) { if (pMgmt == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
@ -50,35 +57,34 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
SSnodeOpt option = {0}; SSnodeOpt option = {0};
smInitOption(pMgmt, &option); smInitOption(pMgmt, &option);
pMgmt->pSnode = sndOpen(pMgmt->path, &option);
if (pMgmt->pSnode == NULL) { code = sndOpenWrapper(pMgmt->path, &option, &pMgmt->pSnode);
dError("failed to open snode since %s", terrstr()); if (code != 0) {
dError("failed to open snode since %s", tstrerror(code));
smClose(pMgmt); smClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("snode-impl", "initialized"); tmsgReportStartup("snode-impl", "initialized");
if (smStartWorker(pMgmt) != 0) { if ((code = smStartWorker(pMgmt)) != 0) {
dError("failed to start snode worker since %s", terrstr()); dError("failed to start snode worker since %s", tstrerror(code));
smClose(pMgmt); smClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("snode-worker", "initialized"); tmsgReportStartup("snode-worker", "initialized");
if (udfcOpen() != 0) { if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in snode"); dError("failed to open udfc in snode since:%s", tstrerror(code));
smClose(pMgmt); smClose(pMgmt);
return -1; return code;
} }
pOutput->pMgmt = pMgmt; pOutput->pMgmt = pMgmt;
return 0; return 0;
} }
static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { return sndInit(pMgmt->pSnode); }
return sndInit(pMgmt->pSnode);
}
SMgmtFunc smGetMgmtFunc() { SMgmtFunc smGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0}; SMgmtFunc mgmtFunc = {0};

60
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -68,17 +68,18 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
} }
int32_t smStartWorker(SSnodeMgmt *pMgmt) { int32_t smStartWorker(SSnodeMgmt *pMgmt) {
int32_t code = 0;
pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *)); pMgmt->writeWroker = taosArrayInit(0, sizeof(SMultiWorker *));
if (pMgmt->writeWroker == NULL) { if (pMgmt->writeWroker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) { for (int32_t i = 0; i < tsNumOfSnodeWriteThreads; i++) {
SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker)); SMultiWorker *pWriteWorker = taosMemoryMalloc(sizeof(SMultiWorker));
if (pWriteWorker == NULL) { if (pWriteWorker == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
SMultiWorkerCfg cfg = { SMultiWorkerCfg cfg = {
@ -87,13 +88,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.fp = smProcessWriteQueue, .fp = smProcessWriteQueue,
.param = pMgmt, .param = pMgmt,
}; };
if (tMultiWorkerInit(pWriteWorker, &cfg) != 0) { if ((code = tMultiWorkerInit(pWriteWorker, &cfg)) != 0) {
dError("failed to start snode-unique worker since %s", terrstr()); dError("failed to start snode-unique worker since %s", tstrerror(code));
return -1; return code;
} }
if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) { if (taosArrayPush(pMgmt->writeWroker, &pWriteWorker) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return code;
} }
} }
@ -105,13 +106,13 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
.param = pMgmt, .param = pMgmt,
}; };
if (tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) { if ((code = tSingleWorkerInit(&pMgmt->streamWorker, &cfg)) != 0) {
dError("failed to start snode shared-worker since %s", terrstr()); dError("failed to start snode shared-worker since %s", tstrerror(code));
return -1; return code;
} }
dDebug("snode workers are initialized"); dDebug("snode workers are initialized");
return 0; return code;
} }
void smStopWorker(SSnodeMgmt *pMgmt) { void smStopWorker(SSnodeMgmt *pMgmt) {
@ -133,17 +134,18 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) { if (code) {
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return code = terrno;
} }
SSnode *pSnode = pMgmt->pSnode; SSnode *pSnode = pMgmt->pSnode;
if (pSnode == NULL) { if (pSnode == NULL) {
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, terrstr(), code = terrno;
dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, tstrerror(code),
TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen); TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return code;
} }
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
@ -154,48 +156,44 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
switch (qtype) { switch (qtype) {
case STREAM_QUEUE: case STREAM_QUEUE:
smPutNodeMsgToStreamQueue(pMgmt, pMsg); code = smPutNodeMsgToStreamQueue(pMgmt, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
smPutNodeMsgToWriteQueue(pMgmt, pMsg); code = smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break; break;
default: default:
terrno = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
return -1; return code;
} }
return 0; return code;
} }
int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToMgmtQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToWriteQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0); SMultiWorker *pWorker = taosArrayGetP(pMgmt->writeWroker, 0);
if (pWorker == NULL) { if (pWorker == NULL) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }
int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smPutNodeMsgToStreamQueue(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SSingleWorker *pWorker = &pMgmt->streamWorker; SSingleWorker *pWorker = &pMgmt->streamWorker;
dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
taosWriteQitem(pWorker->queue, pMsg); return taosWriteQitem(pWorker->queue, pMsg);
return 0;
} }

42
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -54,24 +54,24 @@ typedef struct {
} SWrapperCfg; } SWrapperCfg;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int32_t vgVersion; int32_t vgVersion;
int32_t refCount; int32_t refCount;
int8_t dropped; int8_t dropped;
int8_t failed; int8_t failed;
int8_t disable; int8_t disable;
int32_t diskPrimary; int32_t diskPrimary;
int32_t toVgId; int32_t toVgId;
char *path; char *path;
SVnode *pImpl; SVnode *pImpl;
SMultiWorker pWriteW; SMultiWorker pWriteW;
SMultiWorker pSyncW; SMultiWorker pSyncW;
SMultiWorker pSyncRdW; SMultiWorker pSyncRdW;
SMultiWorker pApplyW; SMultiWorker pApplyW;
STaosQueue *pQueryQ; STaosQueue *pQueryQ;
STaosQueue *pStreamQ; STaosQueue *pStreamQ;
STaosQueue *pFetchQ; STaosQueue *pFetchQ;
STaosQueue *pMultiMgmQ; STaosQueue *pMultiMgmQ;
} SVnodeObj; } SVnodeObj;
typedef struct { typedef struct {
@ -107,9 +107,9 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessArbHeartBeatReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg);
// vmFile.c // vmFile.c
int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt); int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt);
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes); int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes);
// vmWorker.c // vmWorker.c
int32_t vmStartWorker(SVnodeMgmt *pMgmt); int32_t vmStartWorker(SVnodeMgmt *pMgmt);

143
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -19,12 +19,16 @@
#define MAX_CONTENT_LEN 2 * 1024 * 1024 #define MAX_CONTENT_LEN 2 * 1024 * 1024
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { int32_t vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes, SVnodeObj ***ppVnodes) {
taosThreadRwlockRdlock(&pMgmt->lock); taosThreadRwlockRdlock(&pMgmt->lock);
int32_t num = 0; int32_t num = 0;
int32_t size = taosHashGetSize(pMgmt->hash); int32_t size = taosHashGetSize(pMgmt->hash);
SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *)); SVnodeObj **pVnodes = taosMemoryCalloc(size, sizeof(SVnodeObj *));
if (pVnodes == NULL) {
taosThreadRwlockUnlock(&pMgmt->lock);
return TSDB_CODE_OUT_OF_MEMORY;
}
void *pIter = taosHashIterate(pMgmt->hash, NULL); void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) { while (pIter) {
@ -42,8 +46,9 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
taosThreadRwlockUnlock(&pMgmt->lock); taosThreadRwlockUnlock(&pMgmt->lock);
*numOfVnodes = num; *numOfVnodes = num;
*ppVnodes = pVnodes;
return pVnodes; return 0;
} }
static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
@ -52,29 +57,32 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **
*ppCfgs = NULL; *ppCfgs = NULL;
SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes"); SJson *vnodes = tjsonGetObjectItem(pJson, "vnodes");
if (vnodes == NULL) return -1; if (vnodes == NULL) return TSDB_CODE_INVALID_JSON_FORMAT;
int32_t vnodesNum = cJSON_GetArraySize(vnodes); int32_t vnodesNum = cJSON_GetArraySize(vnodes);
if (vnodesNum > 0) { if (vnodesNum > 0) {
pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg)); pCfgs = taosMemoryCalloc(vnodesNum, sizeof(SWrapperCfg));
if (pCfgs == NULL) return -1; if (pCfgs == NULL) return TSDB_CODE_OUT_OF_MEMORY;
} }
for (int32_t i = 0; i < vnodesNum; ++i) { for (int32_t i = 0; i < vnodesNum; ++i) {
SJson *vnode = tjsonGetArrayItem(vnodes, i); SJson *vnode = tjsonGetArrayItem(vnodes, i);
if (vnode == NULL) goto _OVER; if (vnode == NULL) {
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
SWrapperCfg *pCfg = &pCfgs[i]; SWrapperCfg *pCfg = &pCfgs[i];
tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code); tjsonGetInt32ValueFromDouble(vnode, "vgId", pCfg->vgId, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code); tjsonGetInt32ValueFromDouble(vnode, "dropped", pCfg->dropped, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code); tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code); tjsonGetInt32ValueFromDouble(vnode, "diskPrimary", pCfg->diskPrimary, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code); tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code);
if (code < 0) goto _OVER; if (code != 0) goto _OVER;
snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId); snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
} }
@ -98,33 +106,35 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file); code = TAOS_SYSTEM_ERROR(errno);
return 0; dInfo("vnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0;
return code;
} }
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open vnode file:%s since %s", file, terrstr()); dError("failed to open vnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat mnode file:%s since %s", file, terrstr()); dError("failed to fstat mnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
pData = taosMemoryMalloc(size + 1); pData = taosMemoryMalloc(size + 1);
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, pData, size) != size) { if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read vnode file:%s since %s", file, terrstr()); dError("failed to read vnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -132,12 +142,12 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
pJson = tjsonParse(pData); pJson = tjsonParse(pData);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) { if (vmDecodeVnodeList(pJson, pMgmt, ppCfgs, numOfVnodes) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -150,28 +160,36 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read vnode file:%s since %s", file, terrstr()); dError("failed to read vnode file:%s since %s", file, tstrerror(code));
} }
return code; return code;
} }
static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) { static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t numOfVnodes) {
SJson *vnodes = tjsonCreateArray(); int32_t code = 0;
if (vnodes == NULL) return -1; SJson *vnodes = tjsonCreateArray();
if (tjsonAddItemToObject(pJson, "vnodes", vnodes) < 0) return -1; if (vnodes == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if ((code = tjsonAddItemToObject(pJson, "vnodes", vnodes)) < 0) {
tjsonDelete(vnodes);
return code;
};
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i]; SVnodeObj *pVnode = ppVnodes[i];
if (pVnode == NULL) continue; if (pVnode == NULL) continue;
SJson *vnode = tjsonCreateObject(); SJson *vnode = tjsonCreateObject();
if (vnode == NULL) return -1; if (vnode == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId)) < 0) return code;
if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped)) < 0) return code;
if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion)) < 0) return code;
if (tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "diskPrimary", pVnode->diskPrimary)) < 0) return code;
if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1; if (pVnode->toVgId) {
if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1; if ((code = tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId)) < 0) return code;
}
if ((code = tjsonAddItemToArray(vnodes, vnode)) < 0) return code;
} }
return 0; return 0;
@ -185,30 +203,60 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
SVnodeObj **ppVnodes = NULL; SVnodeObj **ppVnodes = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); int32_t nBytes = snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(file)) {
return TSDB_CODE_OUT_OF_RANGE;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (nBytes <= 0 || nBytes >= sizeof(realfile)) {
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (ppVnodes == NULL) goto _OVER; if (code) goto _OVER;
terrno = TSDB_CODE_OUT_OF_MEMORY; // terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) {
if (vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes) != 0) goto _OVER; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = vmEncodeVnodeList(pJson, ppVnodes, numOfVnodes)) != 0) goto _OVER;
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); code = taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER; if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0; code = 0;
dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes); dInfo("succeed to write vnodes file:%s, vnodes:%d", realfile, numOfVnodes);
@ -228,8 +276,7 @@ _OVER:
} }
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, tstrerror(code), numOfVnodes);
dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes);
} }
return code; return code;
} }

25
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -256,8 +256,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
char path[TSDB_FILENAME_LEN] = {0}; char path[TSDB_FILENAME_LEN] = {0};
if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) { if (tDeserializeSCreateVnodeReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG;
return -1;
} }
if (req.learnerReplica == 0) { if (req.learnerReplica == 0) {
@ -298,25 +297,24 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort ||
strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { strcmp(pReplica->fqdn, tsLocalFqdn) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", req.vgId, pReplica->id, pReplica->fqdn, dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode, reason:%s", req.vgId, pReplica->id,
pReplica->port); pReplica->fqdn, pReplica->port, tstrerror(code));
return -1; return code;
} }
if (req.encryptAlgorithm == DND_CA_SM4) { if (req.encryptAlgorithm == DND_CA_SM4) {
if (strlen(tsEncryptKey) == 0) { if (strlen(tsEncryptKey) == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("vgId:%d, failed to create vnode since encrypt key is empty", req.vgId); dError("vgId:%d, failed to create vnode since encrypt key is empty, reason:%s", req.vgId, tstrerror(code));
return -1; return code;
} }
} }
vmGenerateVnodeCfg(&req, &vnodeCfg); vmGenerateVnodeCfg(&req, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) { if ((code = vmTsmaAdjustDays(&vnodeCfg, &req)) < 0) {
dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr()); dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, tstrerror(code));
code = terrno != 0 ? terrno : -1;
goto _OVER; goto _OVER;
} }
@ -327,8 +325,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dError("vgId:%d, already exist", req.vgId); dError("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req); tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_VND_ALREADY_EXIST; code = TSDB_CODE_VND_ALREADY_EXIST;
code = terrno;
return 0; return 0;
} }

101
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -33,6 +33,7 @@ int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
} }
int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t code = 0;
STfs *pTfs = pMgmt->pTfs; STfs *pTfs = pMgmt->pTfs;
int32_t diskId = 0; int32_t diskId = 0;
if (!pTfs) { if (!pTfs) {
@ -59,7 +60,12 @@ int32_t vmAllocPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
// alloc // alloc
int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0}; int32_t disks[TFS_MAX_DISKS_PER_TIER] = {0};
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
return code;
}
for (int32_t v = 0; v < numOfVnodes; v++) { for (int32_t v = 0; v < numOfVnodes; v++) {
SVnodeObj *pVnode = ppVnodes[v]; SVnodeObj *pVnode = ppVnodes[v];
disks[pVnode->diskPrimary] += 1; disks[pVnode->diskPrimary] += 1;
@ -436,6 +442,7 @@ static void *vmCloseVnodeInThread(void *param) {
} }
static void vmCloseVnodes(SVnodeMgmt *pMgmt) { static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
dInfo("start to close all vnodes"); dInfo("start to close all vnodes");
tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->mgmtWorker);
dInfo("vnodes mgmt worker is stopped"); dInfo("vnodes mgmt worker is stopped");
@ -443,7 +450,12 @@ static void vmCloseVnodes(SVnodeMgmt *pMgmt) {
dInfo("vnodes multiple mgmt worker is stopped"); dInfo("vnodes multiple mgmt worker is stopped");
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
int32_t threadNum = tsNumOfCores / 2; int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1; if (threadNum < 1) threadNum = 1;
@ -513,8 +525,14 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
} }
static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return;
}
if (ppVnodes != NULL) { if (ppVnodes != NULL) {
for (int32_t i = 0; i < numOfVnodes; ++i) { for (int32_t i = 0; i < numOfVnodes; ++i) {
@ -549,12 +567,14 @@ static void *vmThreadFp(void *param) {
} }
static int32_t vmInitTimer(SVnodeMgmt *pMgmt) { static int32_t vmInitTimer(SVnodeMgmt *pMgmt) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) { if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) {
dError("failed to create vnode timer thread since %s", strerror(errno)); code = TAOS_SYSTEM_ERROR(errno);
return -1; dError("failed to create vnode timer thread since %s", tstrerror(code));
return code;
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -573,7 +593,10 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = -1; int32_t code = -1;
SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt)); SVnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SVnodeMgmt));
if (pMgmt == NULL) goto _OVER; if (pMgmt == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
pMgmt->pData = pInput->pData; pMgmt->pData = pInput->pData;
pMgmt->path = pInput->path; pMgmt->path = pInput->path;
@ -582,8 +605,18 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue; pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue;
pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize; pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize;
pMgmt->msgCb.mgmt = pMgmt; pMgmt->msgCb.mgmt = pMgmt;
taosThreadRwlockInit(&pMgmt->lock, NULL);
taosThreadMutexInit(&pMgmt->createLock, NULL); code = taosThreadRwlockInit(&pMgmt->lock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = taosThreadMutexInit(&pMgmt->createLock, NULL);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
pMgmt->pTfs = pInput->pTfs; pMgmt->pTfs = pInput->pTfs;
if (pMgmt->pTfs == NULL) { if (pMgmt->pTfs == NULL) {
@ -592,38 +625,39 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
} }
tmsgReportStartup("vnode-tfs", "initialized"); tmsgReportStartup("vnode-tfs", "initialized");
if (walInit() != 0) { if ((code = walInit()) != 0) {
dError("failed to init wal since %s", terrstr()); dError("failed to init wal since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-wal", "initialized"); tmsgReportStartup("vnode-wal", "initialized");
if (syncInit() != 0) { if ((code = syncInit()) != 0) {
dError("failed to open sync since %s", terrstr()); dError("failed to open sync since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-sync", "initialized"); tmsgReportStartup("vnode-sync", "initialized");
if (vnodeInit(tsNumOfCommitThreads) != 0) { if ((code = vnodeInit(tsNumOfCommitThreads)) != 0) {
dError("failed to init vnode since %s", terrstr()); dError("failed to init vnode since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-commit", "initialized"); tmsgReportStartup("vnode-commit", "initialized");
if (vmStartWorker(pMgmt) != 0) { if ((code = vmStartWorker(pMgmt)) != 0) {
dError("failed to init workers since %s", terrstr()); dError("failed to init workers since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-worker", "initialized"); tmsgReportStartup("vnode-worker", "initialized");
if (vmOpenVnodes(pMgmt) != 0) { if ((code = vmOpenVnodes(pMgmt)) != 0) {
dError("failed to open all vnodes since %s", terrstr()); dError("failed to open all vnodes since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
tmsgReportStartup("vnode-vnodes", "initialized"); tmsgReportStartup("vnode-vnodes", "initialized");
if (udfcOpen() != 0) { if ((code = udfcOpen()) != 0) {
dError("failed to open udfc in vnode"); dError("failed to open udfc in vnode since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -633,7 +667,7 @@ _OVER:
if (code == 0) { if (code == 0) {
pOutput->pMgmt = pMgmt; pOutput->pMgmt = pMgmt;
} else { } else {
dError("failed to init vnodes-mgmt since %s", terrstr()); dError("failed to init vnodes-mgmt since %s", tstrerror(code));
vmCleanup(pMgmt); vmCleanup(pMgmt);
} }
@ -683,18 +717,32 @@ static void *vmRestoreVnodeInThread(void *param) {
} }
static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
int32_t code = 0;
int32_t numOfVnodes = 0; int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); SVnodeObj **ppVnodes = NULL;
code = vmGetVnodeListFromHash(pMgmt, &numOfVnodes, &ppVnodes);
if (code != 0) {
dError("failed to get vnode list since %s", tstrerror(code));
return code;
}
int32_t threadNum = tsNumOfCores / 2; int32_t threadNum = tsNumOfCores / 2;
if (threadNum < 1) threadNum = 1; if (threadNum < 1) threadNum = 1;
int32_t vnodesPerThread = numOfVnodes / threadNum + 1; int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread));
if (threads == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t t = 0; t < threadNum; ++t) { for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t; threads[t].threadIndex = t;
threads[t].pMgmt = pMgmt; threads[t].pMgmt = pMgmt;
threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *)); threads[t].ppVnodes = taosMemoryCalloc(vnodesPerThread, sizeof(SVnode *));
if (threads[t].ppVnodes == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
} }
for (int32_t v = 0; v < numOfVnodes; ++v) { for (int32_t v = 0; v < numOfVnodes; ++v) {
@ -717,6 +765,7 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) { if (taosThreadCreate(&pThread->thread, &thAttr, vmRestoreVnodeInThread, pThread) != 0) {
dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno)); dError("thread:%d, failed to create thread to restore vnode since %s", pThread->threadIndex, strerror(errno));
ASSERT(errno == 0);
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
@ -742,6 +791,14 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
} }
return vmInitTimer(pMgmt); return vmInitTimer(pMgmt);
_exit:
for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t];
taosMemoryFree(pThread->ppVnodes);
}
taosMemoryFree(threads);
return code;
} }
static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); } static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); }

70
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -200,26 +200,33 @@ static bool vmDataSpaceSufficient(SVnodeObj *pVnode) {
} }
} }
static int32_t vmAcquireVnodeWrapper(SVnodeMgmt *pMgt, int32_t vgId, SVnodeObj **pNode) {
*pNode = vmAcquireVnode(pMgt, vgId);
if (*pNode == NULL) {
return terrno;
}
return 0;
}
static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) {
int32_t code = 0;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
if (pMsg->contLen < sizeof(SMsgHead)) { if (pMsg->contLen < sizeof(SMsgHead)) {
dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType), dGError("invalid rpc msg with no msg head at pCont. pMsg:%p, type:%s, contLen:%d", pMsg, TMSG_INFO(pMsg->msgType),
pMsg->contLen); pMsg->contLen);
return -1; return TSDB_CODE_INVALID_MSG;
} }
SMsgHead *pHead = pMsg->pCont; SMsgHead *pHead = pMsg->pCont;
int32_t code = 0;
pHead->contLen = ntohl(pHead->contLen); pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId); pHead->vgId = ntohl(pHead->vgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); SVnodeObj *pVnode = NULL;
if (pVnode == NULL) { code = vmAcquireVnodeWrapper(pMgmt, pHead->vgId, &pVnode);
if (code != 0) {
dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); tstrerror(code), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
terrno = (terrno != 0) ? terrno : -1; return code;
return terrno;
} }
switch (qtype) { switch (qtype) {
@ -234,49 +241,45 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
break; break;
case STREAM_QUEUE: case STREAM_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pStreamQ, pMsg); code = taosWriteQitem(pVnode->pStreamQ, pMsg);
break; break;
case FETCH_QUEUE: case FETCH_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pFetchQ, pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg);
break; break;
case WRITE_QUEUE: case WRITE_QUEUE:
if (!vmDataSpaceSufficient(pVnode)) { if (!vmDataSpaceSufficient(pVnode)) {
terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE; code = TSDB_CODE_NO_ENOUGH_DISKSPACE;
code = terrno;
dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code)); dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break; break;
} }
if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) {
terrno = TSDB_CODE_VND_NO_WRITE_AUTH; code = TSDB_CODE_VND_NO_WRITE_AUTH;
code = terrno;
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code)); dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
break; break;
} }
if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) { if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) {
dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg); dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg);
terrno = TSDB_CODE_VND_STOPPED; code = TSDB_CODE_VND_STOPPED;
code = terrno;
break; break;
} }
dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pWriteW.queue, pMsg); code = taosWriteQitem(pVnode->pWriteW.queue, pMsg);
break; break;
case SYNC_QUEUE: case SYNC_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncW.queue, pMsg); code = taosWriteQitem(pVnode->pSyncW.queue, pMsg);
break; break;
case SYNC_RD_QUEUE: case SYNC_RD_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pSyncRdW.queue, pMsg); code = taosWriteQitem(pVnode->pSyncRdW.queue, pMsg);
break; break;
case APPLY_QUEUE: case APPLY_QUEUE:
dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
taosWriteQitem(pVnode->pApplyW.queue, pMsg); code = taosWriteQitem(pVnode->pApplyW.queue, pMsg);
break; break;
default: default:
code = -1; code = TSDB_CODE_INVALID_MSG;
terrno = TSDB_CODE_INVALID_PARA;
break; break;
} }
@ -299,15 +302,13 @@ int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMs
int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutMsgToMultiMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg); dGTrace("msg:%p, put into vnode-multi-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg); return taosWriteQitem(pMgmt->mgmtMultiWorker.queue, pMsg);
return 0;
} }
int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, put into vnode-mgmt queue", pMsg); dGTrace("msg:%p, put into vnode-mgmt queue", pMsg);
taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg); return taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg);
return 0;
} }
int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
@ -317,7 +318,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
pRpc->contLen); pRpc->contLen);
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return TSDB_CODE_INVALID_MSG;
} }
EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM; EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM;
@ -326,7 +327,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
if (code) { if (code) {
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return code;
} }
SMsgHead *pHead = pRpc->pCont; SMsgHead *pHead = pRpc->pCont;
@ -383,6 +384,7 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
} }
int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
int32_t code = 0;
SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl}; SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
@ -398,8 +400,7 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL || if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL ||
pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) { pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue, dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
@ -428,26 +429,27 @@ void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
} }
int32_t vmStartWorker(SVnodeMgmt *pMgmt) { int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
int32_t code = 0;
SQueryAutoQWorkerPool *pQPool = &pMgmt->queryPool; SQueryAutoQWorkerPool *pQPool = &pMgmt->queryPool;
pQPool->name = "vnode-query"; pQPool->name = "vnode-query";
pQPool->min = tsNumOfVnodeQueryThreads; pQPool->min = tsNumOfVnodeQueryThreads;
pQPool->max = tsNumOfVnodeQueryThreads; pQPool->max = tsNumOfVnodeQueryThreads;
if (tQueryAutoQWorkerInit(pQPool) != 0) return -1; if ((code = tQueryAutoQWorkerInit(pQPool)) != 0) return code;
SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool; SAutoQWorkerPool *pStreamPool = &pMgmt->streamPool;
pStreamPool->name = "vnode-stream"; pStreamPool->name = "vnode-stream";
pStreamPool->ratio = tsRatioOfVnodeStreamThreads; pStreamPool->ratio = tsRatioOfVnodeStreamThreads;
if (tAutoQWorkerInit(pStreamPool) != 0) return -1; if ((code = tAutoQWorkerInit(pStreamPool)) != 0) return code;
SWWorkerPool *pFPool = &pMgmt->fetchPool; SWWorkerPool *pFPool = &pMgmt->fetchPool;
pFPool->name = "vnode-fetch"; pFPool->name = "vnode-fetch";
pFPool->max = tsNumOfVnodeFetchThreads; pFPool->max = tsNumOfVnodeFetchThreads;
if (tWWorkerInit(pFPool) != 0) return -1; if ((code = tWWorkerInit(pFPool)) != 0) return code;
SSingleWorkerCfg mgmtCfg = { SSingleWorkerCfg mgmtCfg = {
.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt}; .min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1; if ((code = tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg)) != 0) return code;
int32_t threadNum = 0; int32_t threadNum = 0;
if (tsNumOfCores == 1) { if (tsNumOfCores == 1) {
@ -461,7 +463,7 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
.fp = (FItem)vmProcessMultiMgmtQueue, .fp = (FItem)vmProcessMultiMgmtQueue,
.param = pMgmt}; .param = pMgmt};
if (tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg) != 0) return -1; if ((code = tSingleWorkerInit(&pMgmt->mgmtMultiWorker, &multiMgmtCfg)) != 0) return code;
dDebug("vnode workers are initialized"); dDebug("vnode workers are initialized");
return 0; return 0;

1
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -98,6 +98,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType nType);
int32_t dmMarkWrapper(SMgmtWrapper *pWrapper); int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper); void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode); int32_t dmInitVars(SDnode *pDnode);
int32_t dmInitVarsWrapper(SDnode *pDnode);
void dmClearVars(SDnode *pDnode); void dmClearVars(SDnode *pDnode);
int32_t dmInitModule(SDnode *pDnode); int32_t dmInitModule(SDnode *pDnode);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);

160
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -14,20 +14,22 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
// clang-format off
#include "dmMgmt.h" #include "dmMgmt.h"
#include "audit.h" #include "audit.h"
#include "libs/function/tudf.h" #include "libs/function/tudf.h"
#include "tgrant.h" #include "tgrant.h"
#include "tcompare.h" #include "tcompare.h"
// clang-format on
#define DM_INIT_AUDIT() \ #define DM_INIT_AUDIT() \
do { \ do { \
auditCfg.port = tsMonitorPort; \ auditCfg.port = tsMonitorPort; \
auditCfg.server = tsMonitorFqdn; \ auditCfg.server = tsMonitorFqdn; \
auditCfg.comp = tsMonitorComp; \ auditCfg.comp = tsMonitorComp; \
if (auditInit(&auditCfg) != 0) { \ if ((code = auditInit(&auditCfg)) != 0) { \
return -1; \ return code; \
} \ } \
} while (0) } while (0)
static SDnode globalDnode = {0}; static SDnode globalDnode = {0};
@ -35,10 +37,11 @@ static SDnode globalDnode = {0};
SDnode *dmInstance() { return &globalDnode; } SDnode *dmInstance() { return &globalDnode; }
static int32_t dmCheckRepeatInit(SDnode *pDnode) { static int32_t dmCheckRepeatInit(SDnode *pDnode) {
int32_t code = 0;
if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
dError("env is already initialized"); dError("env is already initialized");
terrno = TSDB_CODE_REPEAT_INIT; code = TSDB_CODE_REPEAT_INIT;
return -1; return code;
} }
return 0; return 0;
} }
@ -58,18 +61,15 @@ static int32_t dmInitMonitor() {
monCfg.port = tsMonitorPort; monCfg.port = tsMonitorPort;
monCfg.server = tsMonitorFqdn; monCfg.server = tsMonitorFqdn;
monCfg.comp = tsMonitorComp; monCfg.comp = tsMonitorComp;
if (monInit(&monCfg) != 0) { if ((code = monInit(&monCfg)) != 0) {
if (terrno != 0) code = terrno; dError("failed to init monitor since %s", tstrerror(code));
goto _exit;
} }
_exit:
if (code) terrno = code;
return code; return code;
} }
static int32_t dmInitAudit() { static int32_t dmInitAudit() {
SAuditCfg auditCfg = {0}; SAuditCfg auditCfg = {0};
int32_t code = 0;
DM_INIT_AUDIT(); DM_INIT_AUDIT();
@ -88,27 +88,34 @@ static bool dmDataSpaceAvailable() {
return true; return true;
} }
static bool dmCheckDiskSpace() { static int32_t dmCheckDiskSpace() {
osUpdate(); osUpdate();
// availability // availability
bool ret = true; int32_t code = 0;
if (!dmDataSpaceAvailable()) { if (!dmDataSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE; code = TSDB_CODE_NO_DISKSPACE;
ret = false; return code;
} }
if (!osLogSpaceAvailable()) { if (!osLogSpaceAvailable()) {
dError("log disk space unavailable, i.e. %s", tsLogDir); dError("log disk space unavailable, i.e. %s", tsLogDir);
terrno = TSDB_CODE_NO_DISKSPACE; code = TSDB_CODE_NO_DISKSPACE;
ret = false; return code;
} }
if (!osTempSpaceAvailable()) { if (!osTempSpaceAvailable()) {
dError("temp disk space unavailable, i.e. %s", tsTempDir); dError("temp disk space unavailable, i.e. %s", tsTempDir);
terrno = TSDB_CODE_NO_DISKSPACE; code = TSDB_CODE_NO_DISKSPACE;
ret = false; return code;
} }
return ret; return code;
} }
int32_t tfsOpenWrapper(SDiskCfg *pCfg, int32_t ndisk, STfs **tfs) {
*tfs = tfsOpen(pCfg, ndisk);
if (*tfs == NULL) {
return terrno;
}
return 0;
}
int32_t dmDiskInit() { int32_t dmDiskInit() {
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0}; SDiskCfg dCfg = {.level = 0, .primary = 1, .disable = 0};
@ -120,10 +127,10 @@ int32_t dmDiskInit() {
numOfDisks = 1; numOfDisks = 1;
} }
pDnode->pTfs = tfsOpen(pDisks, numOfDisks); int32_t code = tfsOpenWrapper(pDisks, numOfDisks, &pDnode->pTfs);
if (pDnode->pTfs == NULL) { if (code != 0) {
dError("failed to init tfs since %s", terrstr()); dError("failed to init tfs since %s", tstrerror(code));
return -1; return code;
} }
return 0; return 0;
} }
@ -146,6 +153,12 @@ static bool dmCheckDataDirVersion() {
return true; return true;
} }
static int32_t dmCheckDataDirVersionWrapper() {
if (!dmCheckDataDirVersion()) {
return TSDB_CODE_INVALID_DATA_FMT;
}
return 0;
}
#if defined(USE_S3) #if defined(USE_S3)
extern int32_t s3Begin(); extern int32_t s3Begin();
@ -156,17 +169,21 @@ extern int8_t tsS3Enabled;
int32_t dmInit() { int32_t dmInit() {
dInfo("start to init dnode env"); dInfo("start to init dnode env");
if (dmDiskInit() != 0) return -1; int32_t code = 0;
if (!dmCheckDataDirVersion()) return -1; if ((code = dmDiskInit()) != 0) return code;
if (!dmCheckDiskSpace()) return -1; if (!dmCheckDataDirVersion()) {
if (dmCheckRepeatInit(dmInstance()) != 0) return -1; code = TSDB_CODE_INVALID_DATA_FMT;
if (dmInitSystem() != 0) return -1; return code;
if (dmInitMonitor() != 0) return -1; }
if (dmInitAudit() != 0) return -1; if ((code = dmCheckDiskSpace()) != 0) return code;
if (dmInitDnode(dmInstance()) != 0) return -1; if ((code = dmCheckRepeatInit(dmInstance())) != 0) return code;
if (InitRegexCache() != 0) return -1; if ((code = dmInitSystem()) != 0) return code;
if ((code = dmInitMonitor()) != 0) return code;
if ((code = dmInitAudit()) != 0) return code;
if ((code = dmInitDnode(dmInstance())) != 0) return code;
if ((code = InitRegexCache() != 0)) return code;
#if defined(USE_S3) #if defined(USE_S3)
if (s3Begin() != 0) return -1; if ((code = s3Begin()) != 0) return code;
#endif #endif
dInfo("dnode env is initialized"); dInfo("dnode env is initialized");
@ -217,6 +234,7 @@ int32_t dmRun() {
} }
static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
@ -224,19 +242,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
switch (ntype) { switch (ntype) {
case MNODE: case MNODE:
terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED; code = TSDB_CODE_MNODE_ALREADY_DEPLOYED;
break; break;
case QNODE: case QNODE:
terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED; code = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
break; break;
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED; code = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
break; break;
default: default:
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
} }
dError("failed to create node since %s", terrstr()); dError("failed to create node since %s", tstrerror(code));
return -1; return code;
} }
dInfo("start to process create-node-request"); dInfo("start to process create-node-request");
@ -244,18 +262,18 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return -1; return code;
} }
taosThreadMutexLock(&pDnode->mutex); taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name); dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg); code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) { if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else { } else {
dInfo("node:%s, has been created", pWrapper->name); dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper); code = dmOpenNode(pWrapper);
@ -271,12 +289,13 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
} }
static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) { if (pWrapper == NULL) {
dError("fail to process alter node type since node not exist"); dError("fail to process alter node type since node not exist");
return -1; return TSDB_CODE_INVALID_MSG;
} }
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
@ -289,16 +308,16 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dInfo("node:%s, checking node role:%d", pWrapper->name, role); dInfo("node:%s, checking node role:%d", pWrapper->name, role);
if (role == TAOS_SYNC_ROLE_VOTER) { if (role == TAOS_SYNC_ROLE_VOTER) {
dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role); dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role);
terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER; code = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
return -1; return code;
} }
} }
if (pWrapper->func.isCatchUpFp != NULL) { if (pWrapper->func.isCatchUpFp != NULL) {
dInfo("node:%s, checking node catch up", pWrapper->name); dInfo("node:%s, checking node catch up", pWrapper->name);
if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) { if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) {
terrno = TSDB_CODE_MNODE_NOT_CATCH_UP; code = TSDB_CODE_MNODE_NOT_CATCH_UP;
return -1; return code;
} }
} }
@ -314,17 +333,17 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype]; pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex); taosThreadMutexUnlock(&pDnode->mutex);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return -1; return code;
} }
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name); dInfo("node:%s, start to create", pWrapper->name);
int32_t code = (*pWrapper->func.createFp)(&input, pMsg); code = (*pWrapper->func.createFp)(&input, pMsg);
if (code != 0) { if (code != 0) {
dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); dError("node:%s, failed to create since %s", pWrapper->name, tstrerror(code));
} else { } else {
dInfo("node:%s, has been created", pWrapper->name); dInfo("node:%s, has been created", pWrapper->name);
code = dmOpenNode(pWrapper); code = dmOpenNode(pWrapper);
@ -340,35 +359,36 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
} }
static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype);
if (pWrapper == NULL) { if (pWrapper == NULL) {
switch (ntype) { switch (ntype) {
case MNODE: case MNODE:
terrno = TSDB_CODE_MNODE_NOT_DEPLOYED; code = TSDB_CODE_MNODE_NOT_DEPLOYED;
break; break;
case QNODE: case QNODE:
terrno = TSDB_CODE_QNODE_NOT_DEPLOYED; code = TSDB_CODE_QNODE_NOT_DEPLOYED;
break; break;
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_NOT_DEPLOYED; code = TSDB_CODE_SNODE_NOT_DEPLOYED;
break; break;
default: default:
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
} }
dError("failed to drop node since %s", terrstr()); dError("failed to drop node since %s", tstrerror(code));
return -1; return terrno = code;
} }
taosThreadMutexLock(&pDnode->mutex); taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to drop", pWrapper->name); dInfo("node:%s, start to drop", pWrapper->name);
int32_t code = (*pWrapper->func.dropFp)(&input, pMsg); code = (*pWrapper->func.dropFp)(&input, pMsg);
if (code != 0) { if (code != 0) {
dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); dError("node:%s, failed to drop since %s", pWrapper->name, tstrerror(code));
} else { } else {
dInfo("node:%s, has been dropped", pWrapper->name); dInfo("node:%s, has been dropped", pWrapper->name);
pWrapper->required = false; pWrapper->required = false;
@ -416,6 +436,4 @@ void dmReportStartup(const char *pName, const char *pDesc) {
int64_t dmGetClusterId() { return globalDnode.data.clusterId; } int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
bool dmReadyForTest() { bool dmReadyForTest() { return dmInstance()->data.dnodeVer > 0; }
return dmInstance()->data.dnodeVer > 0;
}

54
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -42,7 +42,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
int32_t code = -1; int32_t code = -1;
char path[PATH_MAX + 100] = {0}; char path[PATH_MAX + 100] = {0};
if (dmInitVars(pDnode) != 0) { if ((code = dmInitVarsWrapper(pDnode)) != 0) {
goto _OVER; goto _OVER;
} }
@ -65,26 +65,31 @@ int32_t dmInitDnode(SDnode *pDnode) {
snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name); snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name);
pWrapper->path = taosStrdup(path); pWrapper->path = taosStrdup(path);
if (pWrapper->path == NULL) { if (pWrapper->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
pWrapper->required = dmRequireNode(pDnode, pWrapper); pWrapper->required = dmRequireNode(pDnode, pWrapper);
} }
pDnode->lockfile = dmCheckRunning(tsDataDir); code = dmCheckRunning(tsDataDir, &pDnode->lockfile);
if (pDnode->lockfile == NULL) { if (code != 0) {
goto _OVER; goto _OVER;
} }
if (dmInitModule(pDnode) != 0) {
if ((code = dmInitModule(pDnode)) != 0) {
goto _OVER; goto _OVER;
} }
indexInit(tsNumOfCommitThreads); indexInit(tsNumOfCommitThreads);
streamMetaInit(); streamMetaInit();
dmInitStatusClient(pDnode); if ((code = dmInitStatusClient(pDnode)) != 0) {
dmInitSyncClient(pDnode); goto _OVER;
}
if ((code = dmInitSyncClient(pDnode)) != 0) {
goto _OVER;
}
dmReportStartup("dnode-transport", "initialized"); dmReportStartup("dnode-transport", "initialized");
dDebug("dnode is created, ptr:%p", pDnode); dDebug("dnode is created, ptr:%p", pDnode);
@ -94,7 +99,7 @@ _OVER:
if (code != 0 && pDnode != NULL) { if (code != 0 && pDnode != NULL) {
dmClearVars(pDnode); dmClearVars(pDnode);
pDnode = NULL; pDnode = NULL;
dError("failed to create dnode since %s", terrstr()); dError("failed to create dnode since %s", tstrerror(code));
} }
return code; return code;
@ -122,7 +127,15 @@ void dmCleanupDnode(SDnode *pDnode) {
dDebug("dnode is closed, ptr:%p", pDnode); dDebug("dnode is closed, ptr:%p", pDnode);
} }
int32_t dmInitVarsWrapper(SDnode *pDnode) {
int32_t code = dmInitVars(pDnode);
if (code == -1) {
return terrno;
}
return 0;
}
int32_t dmInitVars(SDnode *pDnode) { int32_t dmInitVars(SDnode *pDnode) {
int32_t code = 0;
SDnodeData *pData = &pDnode->data; SDnodeData *pData = &pDnode->data;
pData->dnodeId = 0; pData->dnodeId = 0;
pData->clusterId = 0; pData->clusterId = 0;
@ -138,21 +151,21 @@ int32_t dmInitVars(SDnode *pDnode) {
taosMemoryFreeClear(machineId); taosMemoryFreeClear(machineId);
} else { } else {
#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG) #if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE; code = TSDB_CODE_DNODE_NO_MACHINE_CODE;
return -1; return terrno = code;
#endif #endif
} }
pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pData->dnodeHash == NULL) { if (pData->dnodeHash == NULL) {
dError("failed to init dnode hash"); dError("failed to init dnode hash");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno = code;
} }
if (dmReadEps(pData) != 0) { if ((code = dmReadEps(pData)) != 0) {
dError("failed to read file since %s", terrstr()); dError("failed to read file since %s", tstrerror(code));
return -1; return code;
} }
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
@ -274,22 +287,21 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
} else { } else {
switch (pWrapper->ntype) { switch (pWrapper->ntype) {
case MNODE: case MNODE:
terrno = TSDB_CODE_MNODE_NOT_FOUND; code = TSDB_CODE_MNODE_NOT_FOUND;
break; break;
case QNODE: case QNODE:
terrno = TSDB_CODE_QNODE_NOT_FOUND; code = TSDB_CODE_QNODE_NOT_FOUND;
break; break;
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_NOT_FOUND; code = TSDB_CODE_SNODE_NOT_FOUND;
break; break;
case VNODE: case VNODE:
terrno = TSDB_CODE_VND_STOPPED; code = TSDB_CODE_VND_STOPPED;
break; break;
default: default:
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
break; break;
} }
code = -1;
} }
taosThreadRwlockUnlock(&pWrapper->lock); taosThreadRwlockUnlock(&pWrapper->lock);

47
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -17,12 +17,13 @@
#include "dmMgmt.h" #include "dmMgmt.h"
int32_t dmOpenNode(SMgmtWrapper *pWrapper) { int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
SDnode *pDnode = pWrapper->pDnode; SDnode *pDnode = pWrapper->pDnode;
if (taosMkDir(pWrapper->path) != 0) { if (taosMkDir(pWrapper->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr()); dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code));
return -1; return code;
} }
SMgmtOutputOpt output = {0}; SMgmtOutputOpt output = {0};
@ -30,9 +31,9 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to open", pWrapper->name); dInfo("node:%s, start to open", pWrapper->name);
tmsgSetDefault(&input.msgCb); tmsgSetDefault(&input.msgCb);
if ((*pWrapper->func.openFp)(&input, &output) != 0) { if ((code = (*pWrapper->func.openFp)(&input, &output)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
dInfo("node:%s, has been opened", pWrapper->name); dInfo("node:%s, has been opened", pWrapper->name);
pWrapper->deployed = true; pWrapper->deployed = true;
@ -46,11 +47,12 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
} }
int32_t dmStartNode(SMgmtWrapper *pWrapper) { int32_t dmStartNode(SMgmtWrapper *pWrapper) {
int32_t code = 0;
if (pWrapper->func.startFp != NULL) { if (pWrapper->func.startFp != NULL) {
dDebug("node:%s, start to start", pWrapper->name); dDebug("node:%s, start to start", pWrapper->name);
if ((*pWrapper->func.startFp)(pWrapper->pMgmt) != 0) { if ((code = (*pWrapper->func.startFp)(pWrapper->pMgmt)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
dDebug("node:%s, has been started", pWrapper->name); dDebug("node:%s, has been started", pWrapper->name);
} }
@ -86,12 +88,13 @@ void dmCloseNode(SMgmtWrapper *pWrapper) {
} }
static int32_t dmOpenNodes(SDnode *pDnode) { static int32_t dmOpenNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (dmOpenNode(pWrapper) != 0) { if ((code = dmOpenNode(pWrapper)) != 0) {
dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); dError("node:%s, failed to open since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
} }
@ -100,12 +103,13 @@ static int32_t dmOpenNodes(SDnode *pDnode) {
} }
static int32_t dmStartNodes(SDnode *pDnode) { static int32_t dmStartNodes(SDnode *pDnode) {
int32_t code = 0;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
if (!pWrapper->required) continue; if (!pWrapper->required) continue;
if (dmStartNode(pWrapper) != 0) { if ((code = dmStartNode(pWrapper)) != 0) {
dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); dError("node:%s, failed to start since %s", pWrapper->name, tstrerror(code));
return -1; return code;
} }
} }
@ -129,19 +133,20 @@ static void dmCloseNodes(SDnode *pDnode) {
} }
int32_t dmRunDnode(SDnode *pDnode) { int32_t dmRunDnode(SDnode *pDnode) {
int32_t code = 0;
int32_t count = 0; int32_t count = 0;
if (dmOpenNodes(pDnode) != 0) { if ((code = dmOpenNodes(pDnode)) != 0) {
dError("failed to open nodes since %s", terrstr()); dError("failed to open nodes since %s", tstrerror(code));
dmCloseNodes(pDnode); dmCloseNodes(pDnode);
return -1; return code;
} }
if (dmStartNodes(pDnode) != 0) { if ((code = dmStartNodes(pDnode)) != 0) {
dError("failed to start nodes since %s", terrstr()); dError("failed to start nodes since %s", tstrerror(code));
dmSetStatus(pDnode, DND_STAT_STOPPED); dmSetStatus(pDnode, DND_STAT_STOPPED);
dmStopNodes(pDnode); dmStopNodes(pDnode);
dmCloseNodes(pDnode); dmCloseNodes(pDnode);
return -1; return code;
} }
while (1) { while (1) {

68
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -39,9 +39,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
if (msgFp == NULL) { if (msgFp == NULL) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED; // terrno = TSDB_CODE_MSG_NOT_PROCESSED;
dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType));
return -1; return TSDB_CODE_MSG_NOT_PROCESSED;
} }
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
@ -54,14 +54,15 @@ static bool dmFailFastFp(tmsg_t msgType) {
return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES; return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES;
} }
static void dmConvertErrCode(tmsg_t msgType) { static int32_t dmConvertErrCode(tmsg_t msgType, int32_t code) {
if (terrno != TSDB_CODE_APP_IS_STOPPING) { if (code != TSDB_CODE_APP_IS_STOPPING) {
return; return code;
} }
if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) || if ((msgType > TDMT_VND_MSG_MIN && msgType < TDMT_VND_MSG_MAX) ||
(msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) { (msgType > TDMT_SCH_MSG_MIN && msgType < TDMT_SCH_MSG_MAX)) {
terrno = TSDB_CODE_VND_STOPPED; code = TSDB_CODE_VND_STOPPED;
} }
return code;
} }
static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) { static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite)); SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite));
@ -99,14 +100,14 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
int32_t svrVer = 0; int32_t svrVer = 0;
taosVersionStrToInt(version, &svrVer); taosVersionStrToInt(version, &svrVer);
if (0 != taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) { if ((code = taosCheckVersionCompatible(pRpc->info.cliVer, svrVer, 3)) != 0) {
dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer); dError("Version not compatible, cli ver: %d, svr ver: %d", pRpc->info.cliVer, svrVer);
goto _OVER; goto _OVER;
} }
bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp); bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp);
if (isForbidden) { if (isForbidden) {
terrno = TSDB_CODE_IP_NOT_IN_WHITE_LIST; code = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
goto _OVER; goto _OVER;
} }
@ -119,7 +120,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
case TDMT_SCH_FETCH_RSP: case TDMT_SCH_FETCH_RSP:
case TDMT_SCH_MERGE_FETCH_RSP: case TDMT_SCH_MERGE_FETCH_RSP:
case TDMT_VND_SUBMIT_RSP: case TDMT_VND_SUBMIT_RSP:
qWorkerProcessRspMsg(NULL, NULL, pRpc, 0); code = qWorkerProcessRspMsg(NULL, NULL, pRpc, 0);
return; return;
case TDMT_MND_STATUS_RSP: case TDMT_MND_STATUS_RSP:
if (pEpSet != NULL) { if (pEpSet != NULL) {
@ -148,32 +149,32 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
return; return;
} else { } else {
if (pDnode->status == DND_STAT_INIT) { if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
} else { } else {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
} }
goto _OVER; goto _OVER;
} }
} }
} else { } else {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
goto _OVER; goto _OVER;
} }
if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) { if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) {
dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType)); dGError("msg:%p, type:%s pCont is NULL", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN; code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER; goto _OVER;
} else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) && } else if ((pRpc->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRpc->code == TSDB_CODE_RPC_BROKEN_LINK) &&
(!IsReq(pRpc)) && (pRpc->pCont == NULL)) { (!IsReq(pRpc)) && (pRpc->pCont == NULL)) {
dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code)); dGError("msg:%p, type:%s pCont is NULL, err: %s", pRpc, TMSG_INFO(pRpc->msgType), tstrerror(pRpc->code));
terrno = pRpc->code; code = pRpc->code;
goto _OVER; goto _OVER;
} }
if (pHandle->defaultNtype == NODE_END) { if (pHandle->defaultNtype == NODE_END) {
dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType)); dGError("msg:%p, type:%s not processed since no handle", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
goto _OVER; goto _OVER;
} }
@ -197,12 +198,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
} }
} else { } else {
dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType)); dGError("msg:%p, type:%s contLen is 0", pRpc, TMSG_INFO(pRpc->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN; code = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER; goto _OVER;
} }
} }
if (dmMarkWrapper(pWrapper) != 0) { if ((code = dmMarkWrapper(pWrapper)) != 0) {
pWrapper = NULL; pWrapper = NULL;
goto _OVER; goto _OVER;
} }
@ -221,12 +222,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
dmConvertErrCode(pRpc->msgType); code = dmConvertErrCode(pRpc->msgType, code);
if (terrno != 0) code = terrno;
if (pMsg) { if (pMsg) {
dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code));
} else { } else {
dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr()); dGTrace("msg:%p, failed to process empty msg since %s", pMsg, tstrerror(code));
} }
if (IsReq(pRpc)) { if (IsReq(pRpc)) {
@ -280,17 +280,19 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
} }
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) { if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) { if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
} else { } else {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
} }
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
return -1; pMsg->info.handle);
return code;
} else { } else {
pMsg->info.handle = 0; pMsg->info.handle = 0;
rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL);
@ -298,17 +300,19 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
} }
} }
static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t code = 0;
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();
if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) { if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG_MIN) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
if (pDnode->status == DND_STAT_INIT) { if (pDnode->status == DND_STAT_INIT) {
terrno = TSDB_CODE_APP_IS_STARTING; code = TSDB_CODE_APP_IS_STARTING;
} else { } else {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
} }
dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), tstrerror(code),
return -1; pMsg->info.handle);
return code;
} else { } else {
rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL); rpcSendRequest(pDnode->trans.syncRpc, pEpSet, pMsg, NULL);
return 0; return 0;
@ -426,7 +430,7 @@ int32_t dmInitStatusClient(SDnode *pDnode) {
pTrans->statusRpc = rpcOpen(&rpcInit); pTrans->statusRpc = rpcOpen(&rpcInit);
if (pTrans->statusRpc == NULL) { if (pTrans->statusRpc == NULL) {
dError("failed to init dnode rpc status client"); dError("failed to init dnode rpc status client");
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
dDebug("dnode rpc status client is initialized"); dDebug("dnode rpc status client is initialized");
@ -471,7 +475,7 @@ int32_t dmInitSyncClient(SDnode *pDnode) {
pTrans->syncRpc = rpcOpen(&rpcInit); pTrans->syncRpc = rpcOpen(&rpcInit);
if (pTrans->syncRpc == NULL) { if (pTrans->syncRpc == NULL) {
dError("failed to init dnode rpc sync client"); dError("failed to init dnode rpc sync client");
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
dDebug("dnode rpc sync client is initialized"); dDebug("dnode rpc sync client is initialized");
@ -521,7 +525,7 @@ int32_t dmInitServer(SDnode *pDnode) {
pTrans->serverRpc = rpcOpen(&rpcInit); pTrans->serverRpc = rpcOpen(&rpcInit);
if (pTrans->serverRpc == NULL) { if (pTrans->serverRpc == NULL) {
dError("failed to init dnode rpc server"); dError("failed to init dnode rpc server");
return -1; return terrno;
} }
dDebug("dnode rpc server is initialized"); dDebug("dnode rpc server is initialized");

33
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -145,20 +145,20 @@ typedef struct {
} SDnodeData; } SDnodeData;
typedef struct { typedef struct {
const char *path; const char *path;
const char *name; const char *name;
STfs *pTfs; STfs *pTfs;
SDnodeData *pData; SDnodeData *pData;
SMsgCb msgCb; SMsgCb msgCb;
ProcessCreateNodeFp processCreateNodeFp; ProcessCreateNodeFp processCreateNodeFp;
ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessAlterNodeTypeFp processAlterNodeTypeFp;
ProcessDropNodeFp processDropNodeFp; ProcessDropNodeFp processDropNodeFp;
SendMonitorReportFp sendMonitorReportFp; SendMonitorReportFp sendMonitorReportFp;
SendAuditRecordsFp sendAuditRecordFp; SendAuditRecordsFp sendAuditRecordFp;
GetVnodeLoadsFp getVnodeLoadsFp; GetVnodeLoadsFp getVnodeLoadsFp;
GetVnodeLoadsFp getVnodeLoadsLiteFp; GetVnodeLoadsFp getVnodeLoadsLiteFp;
GetMnodeLoadsFp getMnodeLoadsFp; GetMnodeLoadsFp getMnodeLoadsFp;
GetQnodeLoadsFp getQnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp;
} SMgmtInputOpt; } SMgmtInputOpt;
typedef struct { typedef struct {
@ -203,9 +203,10 @@ void *dmSetMgmtHandle(SArray *pArray, tmsg_t msgType, void *nodeMsgFp, boo
void dmGetMonitorSystemInfo(SMonSysInfo *pInfo); void dmGetMonitorSystemInfo(SMonSysInfo *pInfo);
// dmFile.c // dmFile.c
int32_t dmReadFile(const char *path, const char *name, bool *pDeployed); int32_t dmReadFile(const char *path, const char *name, bool *pDeployed);
int32_t dmWriteFile(const char *path, const char *name, bool deployed); int32_t dmWriteFile(const char *path, const char *name, bool deployed);
TdFilePtr dmCheckRunning(const char *dataDir); int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile);
//int32_t dmCheckRunningWrapper(const char *dataDir, TdFilePtr *pFile);
// dmodule.c // dmodule.c
int32_t dmInitDndInfo(SDnodeData *pData); int32_t dmInitDndInfo(SDnodeData *pData);

122
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -95,18 +95,18 @@ static int32_t dmDecodeEps(SJson *pJson, SDnodeData *pData) {
} }
int dmOccurrences(char *str, char *toSearch) { int dmOccurrences(char *str, char *toSearch) {
int count = 0; int count = 0;
char *ptr = str; char *ptr = str;
while ((ptr = strstr(ptr, toSearch)) != NULL) { while ((ptr = strstr(ptr, toSearch)) != NULL) {
count++; count++;
ptr++; ptr++;
} }
return count; return count;
} }
void dmSplitStr(char** arr, char* str, const char* del) { void dmSplitStr(char **arr, char *str, const char *del) {
char *lasts; char *lasts;
char* s = strsep(&str, del); char *s = strsep(&str, del);
while (s != NULL) { while (s != NULL) {
*arr++ = s; *arr++ = s;
s = strsep(&str, del); s = strsep(&str, del);
@ -132,11 +132,10 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("dnode file:%s not exist", file); dInfo("dnode file:%s not exist", file);
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
if(strlen(tsEncryptAlgorithm) > 0){ if (strlen(tsEncryptAlgorithm) > 0) {
if(strcmp(tsEncryptAlgorithm, "sm4") == 0) { if (strcmp(tsEncryptAlgorithm, "sm4") == 0) {
pData->encryptAlgorigthm = DND_CA_SM4; pData->encryptAlgorigthm = DND_CA_SM4;
} } else {
else{
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm); dError("invalid tsEncryptAlgorithm:%s", tsEncryptAlgorithm);
goto _OVER; goto _OVER;
@ -144,45 +143,45 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("start to parse encryptScope:%s", tsEncryptScope); dInfo("start to parse encryptScope:%s", tsEncryptScope);
int32_t scopeLen = strlen(tsEncryptScope); int32_t scopeLen = strlen(tsEncryptScope);
if(scopeLen == 0){ if (scopeLen == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dError("invalid tsEncryptScope:%s", tsEncryptScope); dError("invalid tsEncryptScope:%s", tsEncryptScope);
goto _OVER; goto _OVER;
} }
char* tmp = taosMemoryMalloc(scopeLen + 1); char *tmp = taosMemoryMalloc(scopeLen + 1);
memset(tmp, 0, scopeLen + 1); memset(tmp, 0, scopeLen + 1);
memcpy(tmp, tsEncryptScope, scopeLen); memcpy(tmp, tsEncryptScope, scopeLen);
int32_t count = dmOccurrences(tmp, ","); int32_t count = dmOccurrences(tmp, ",");
char** array = taosMemoryMalloc(sizeof(char*) * (count + 1)); char **array = taosMemoryMalloc(sizeof(char *) * (count + 1));
memset(array, 0, sizeof(char*) * (count + 1)); memset(array, 0, sizeof(char *) * (count + 1));
dmSplitStr(array, tmp, ","); dmSplitStr(array, tmp, ",");
for(int32_t i = 0; i < count + 1; i++){ for (int32_t i = 0; i < count + 1; i++) {
char* str = *(array + i); char *str = *(array + i);
bool success = false; bool success = false;
if(strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "tsdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_TSDB; pData->encryptScope |= DND_CS_TSDB;
success = true; success = true;
} }
if(strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "vnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_VNODE_WAL; pData->encryptScope |= DND_CS_VNODE_WAL;
success = true; success = true;
} }
if(strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "sdb") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_SDB; pData->encryptScope |= DND_CS_SDB;
success = true; success = true;
} }
if(strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0){ if (strcasecmp(str, "mnode_wal") == 0 || strcasecmp(str, "all") == 0) {
pData->encryptScope |= DND_CS_MNODE_WAL; pData->encryptScope |= DND_CS_MNODE_WAL;
success = true; success = true;
} }
if(!success){ if (!success) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG; terrno = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
taosMemoryFree(tmp); taosMemoryFree(tmp);
taosMemoryFree(array); taosMemoryFree(array);
@ -196,7 +195,7 @@ int32_t dmReadEps(SDnodeData *pData) {
dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope); dInfo("set tsCryptAlgorithm:%s, tsCryptScope:%s from cfg", tsEncryptAlgorithm, tsEncryptScope);
} }
#endif #endif
code = 0; code = 0;
goto _OVER; goto _OVER;
@ -204,26 +203,26 @@ int32_t dmReadEps(SDnodeData *pData) {
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr()); dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr()); dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
@ -232,12 +231,11 @@ int32_t dmReadEps(SDnodeData *pData) {
pJson = tjsonParse(content); pJson = tjsonParse(content);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (dmDecodeEps(pJson, pData) < 0) { if ((code = dmDecodeEps(pJson, pData)) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -251,7 +249,7 @@ _OVER:
if (code != 0) { if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, terrstr());
return code; return terrno = code;
} }
if (taosArrayGetSize(pData->dnodeEps) == 0) { if (taosArrayGetSize(pData->dnodeEps) == 0) {
@ -261,8 +259,8 @@ _OVER:
taosArrayPush(pData->dnodeEps, &dnodeEp); taosArrayPush(pData->dnodeEps, &dnodeEp);
} }
if (dmReadDnodePairs(pData) != 0) { if ((code = dmReadDnodePairs(pData)) != 0) {
return -1; return terrno = code;
} }
dDebug("reset dnode list on startup"); dDebug("reset dnode list on startup");
@ -270,8 +268,8 @@ _OVER:
if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) { if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) {
dError("localEp %s different with %s and need to be reconfigured", tsLocalEp, file); dError("localEp %s different with %s and need to be reconfigured", tsLocalEp, file);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; return terrno = code;
} }
return code; return code;
@ -308,7 +306,7 @@ static int32_t dmEncodeEps(SJson *pJson, SDnodeData *pData) {
} }
int32_t dmWriteEps(SDnodeData *pData) { int32_t dmWriteEps(SDnodeData *pData) {
int32_t code = -1; int32_t code = 0;
char *buffer = NULL; char *buffer = NULL;
SJson *pJson = NULL; SJson *pJson = NULL;
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
@ -317,26 +315,30 @@ int32_t dmWriteEps(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
terrno = TSDB_CODE_OUT_OF_MEMORY; // if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmInitDndInfo(pData), NULL, _OVER);
if ((code == dmInitDndInfo(pData)) != 0) goto _OVER;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
pData->engineVer = tsVersion; pData->engineVer = tsVersion;
if (dmEncodeEps(pJson, pData) != 0) goto _OVER;
TAOS_CHECK_GOTO(dmEncodeEps(pJson, pData), NULL, _OVER); // dmEncodeEps(pJson, pData) != 0) goto _OVER;
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _OVER);
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
if (taosFsyncFile(pFile) < 0) goto _OVER; if (taosFsyncFile(pFile) < 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
taosCloseFile(&pFile); taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER; if (taosRenameFile(file, realfile) != 0) TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _OVER);
code = 0; code = 0;
pData->updateTime = taosGetTimestampMs(); pData->updateTime = taosGetTimestampMs();
@ -349,8 +351,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, tstrerror(code), pData->dnodeVer);
dError("failed to write dnode file:%s since %s, dnodeVer:%" PRId64, realfile, terrstr(), pData->dnodeVer);
} }
return code; return code;
} }
@ -592,28 +593,29 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file); code = TAOS_SYSTEM_ERROR(errno);
dDebug("dnode file:%s not exist, reason:%s", file, tstrerror(code));
code = 0; code = 0;
goto _OVER; goto _OVER;
} }
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr()); dError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr()); dError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -627,12 +629,13 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
pJson = tjsonParse(content); pJson = tjsonParse(content);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair)); pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair));
if (pData->oldDnodeEps == NULL) { if (pData->oldDnodeEps == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
dError("failed to calloc dnodeEp array since %s", strerror(errno)); dError("failed to calloc dnodeEp array since %s", strerror(errno));
goto _OVER; goto _OVER;
} }
@ -640,7 +643,8 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
if (dmDecodeEpPairs(pJson, pData) < 0) { if (dmDecodeEpPairs(pJson, pData) < 0) {
taosArrayDestroy(pData->oldDnodeEps); taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL; pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -653,7 +657,7 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, tstrerror(code));
return code; return code;
} }
@ -680,8 +684,8 @@ _OVER:
pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id); pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id);
taosArrayDestroy(pData->oldDnodeEps); taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL; pData->oldDnodeEps = NULL;
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; return code;
} }
} }
} }

346
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -14,23 +14,23 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "dmUtil.h"
#include "tjson.h"
#include "tgrant.h"
#include "crypt.h" #include "crypt.h"
#include "dmUtil.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "tgrant.h"
#include "tjson.h"
#define MAXLEN 1024 #define MAXLEN 1024
#define DM_KEY_INDICATOR "this indicator!" #define DM_KEY_INDICATOR "this indicator!"
#define DM_ENCRYPT_CODE_FILE "encryptCode.cfg" #define DM_ENCRYPT_CODE_FILE "encryptCode.cfg"
#define DM_CHECK_CODE_FILE "checkCode.bin" #define DM_CHECK_CODE_FILE "checkCode.bin"
static int32_t dmDecodeFile(SJson *pJson, bool *deployed) { static int32_t dmDecodeFile(SJson *pJson, bool *deployed) {
int32_t code = 0; int32_t code = 0;
int32_t value = 0; int32_t value = 0;
tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code); tjsonGetInt32ValueFromDouble(pJson, "deployed", value, code);
if (code < 0) return -1; if (code < 0) return code;
*deployed = (value != 0); *deployed = (value != 0);
return code; return code;
@ -42,7 +42,11 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char *content = NULL; char *content = NULL;
SJson *pJson = NULL; SJson *pJson = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
if (taosStatFile(file, NULL, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file); dInfo("file:%s not exist", file);
@ -52,27 +56,27 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", file, terrstr()); dError("failed to open file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat file:%s since %s", file, terrstr()); dError("failed to fstat file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read file:%s since %s", file, terrstr()); dError("failed to read file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -80,12 +84,12 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
pJson = tjsonParse(content); pJson = tjsonParse(content);
if (pJson == NULL) { if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
if (dmDecodeFile(pJson, pDeployed) < 0) { if (dmDecodeFile(pJson, pDeployed) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER; goto _OVER;
} }
@ -98,13 +102,13 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, tstrerror(code));
} }
return code; return code;
} }
static int32_t dmEncodeFile(SJson *pJson, bool deployed) { static int32_t dmEncodeFile(SJson *pJson, bool deployed) {
if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "deployed", deployed) < 0) return TSDB_CODE_INVALID_JSON_FORMAT;
return 0; return 0;
} }
@ -115,26 +119,57 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t nBytes = snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
nBytes = snprintf(realfile, sizeof(realfile), "%s%s%s.json", path, TD_DIRSEP, name);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
code = TSDB_CODE_OUT_OF_BUFFER;
goto _OVER;
}
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER; if (pJson == NULL) {
if (dmEncodeFile(pJson, deployed) != 0) goto _OVER; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if ((code = dmEncodeFile(pJson, deployed)) != 0) goto _OVER;
buffer = tjsonToString(pJson); buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER; if (buffer == NULL) {
terrno = 0; code = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(buffer); int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER; if (taosWriteFile(pFile, buffer, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosCloseFile(&pFile) != 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
code = 0; code = 0;
dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed); dInfo("succeed to write file:%s, deloyed:%d", realfile, deployed);
@ -145,54 +180,58 @@ _OVER:
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write file:%s since %s, deloyed:%d", realfile, tstrerror(code), deployed);
dError("failed to write file:%s since %s, deloyed:%d", realfile, terrstr(), deployed);
} }
return code; return code;
} }
TdFilePtr dmCheckRunning(const char *dataDir) { int32_t dmCheckRunning(const char *dataDir, TdFilePtr *pFile) {
char filepath[PATH_MAX] = {0}; int32_t code = 0;
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP); snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC); *pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_CLOEXEC);
if (pFile == NULL) { if (*pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open file:%s since %s", filepath, terrstr()); dError("failed to open file:%s since %s", filepath, tstrerror(code));
return NULL; return code;
} }
int32_t retryTimes = 0; int32_t retryTimes = 0;
int32_t ret = 0; int32_t ret = 0;
do { do {
ret = taosLockFile(pFile); ret = taosLockFile(*pFile);
if (ret == 0) break; if (ret == 0) break;
terrno = TAOS_SYSTEM_ERROR(errno);
code = TAOS_SYSTEM_ERROR(errno);
taosMsleep(1000); taosMsleep(1000);
retryTimes++; retryTimes++;
dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); dError("failed to lock file:%s since %s, retryTimes:%d", filepath, tstrerror(code), retryTimes);
} while (retryTimes < 12); } while (retryTimes < 12);
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(pFile);
return NULL; *pFile = NULL;
return code;
} }
terrno = 0;
dDebug("lock file:%s to prevent repeated starts", filepath); dDebug("lock file:%s to prevent repeated starts", filepath);
return pFile; return code;
} }
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode); extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool toLogFile){ static int32_t dmWriteCheckCodeFile(char *file, char *realfile, char *key, bool toLogFile) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char *result = NULL; char *result = NULL;
int32_t code = -1; int32_t code = -1;
int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR)); int32_t len = ENCRYPTED_LEN(sizeof(DM_KEY_INDICATOR));
result = taosMemoryMalloc(len); result = taosMemoryMalloc(len);
if (result == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SCryptOpts opts; SCryptOpts opts;
strncpy(opts.key, key, ENCRYPT_KEY_LEN); strncpy(opts.key, key, ENCRYPT_KEY_LEN);
@ -203,48 +242,81 @@ static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool
CBC_Encrypt(&opts); CBC_Encrypt(&opts);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosWriteFile(pFile, opts.result, len) <= 0) goto _OVER; if (taosWriteFile(pFile, opts.result, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosFsyncFile(pFile) < 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosCloseFile(&pFile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write checkCode file:%s", realfile); encryptDebug("succeed to write checkCode file:%s", realfile);
code = 0; code = 0;
_OVER: _OVER:
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if(result != NULL) taosMemoryFree(result); if (result != NULL) taosMemoryFree(result);
return code; return code;
} }
static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode, bool toLogFile){ static int32_t dmWriteEncryptCodeFile(char *file, char *realfile, char *encryptCode, bool toLogFile) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
int32_t code = -1; int32_t code = -1;
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) goto _OVER; if (pFile == NULL) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
int32_t len = strlen(encryptCode); int32_t len = strlen(encryptCode);
if (taosWriteFile(pFile, encryptCode, len) <= 0) goto _OVER; if (taosWriteFile(pFile, encryptCode, len) <= 0) {
if (taosFsyncFile(pFile) < 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile); if (taosCloseFile(&pFile) != 0) {
if (taosRenameFile(file, realfile) != 0) goto _OVER; code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosRenameFile(file, realfile) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
encryptDebug("succeed to write encryptCode file:%s", realfile); encryptDebug("succeed to write encryptCode file:%s", realfile);
code = 0; code = 0;
_OVER: _OVER:
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
return code; return code;
} }
static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){ static int32_t dmCompareEncryptKey(char *file, char *key, bool toLogFile) {
char *content = NULL; char *content = NULL;
int64_t size = 0; int64_t size = 0;
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
@ -253,31 +325,31 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to open dnode file:%s since %s", file, terrstr()); encryptError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to fstat dnode file:%s since %s", file, terrstr()); encryptError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size); content = taosMemoryMalloc(size);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to read dnode file:%s since %s", file, terrstr()); encryptError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
encryptDebug("succeed to read checkCode file:%s", file); encryptDebug("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size); int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len); result = taosMemoryMalloc(len);
@ -289,8 +361,8 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
opts.unitLen = 16; opts.unitLen = 16;
CBC_Decrypt(&opts); CBC_Decrypt(&opts);
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) { if (strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED; code = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
encryptError("failed to compare decrypted result"); encryptError("failed to compare decrypted result");
goto _OVER; goto _OVER;
} }
@ -298,9 +370,9 @@ static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
encryptDebug("succeed to compare checkCode file:%s", file); encryptDebug("succeed to compare checkCode file:%s", file);
code = 0; code = 0;
_OVER: _OVER:
if(result != NULL) taosMemoryFree(result); if (result != NULL) taosMemoryFree(result);
if(content != NULL) taosMemoryFree(content); if (content != NULL) taosMemoryFree(content);
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
return code; return code;
} }
@ -319,40 +391,57 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
char checkFile[PATH_MAX] = {0}; char checkFile[PATH_MAX] = {0};
char realCheckFile[PATH_MAX] = {0}; char realCheckFile[PATH_MAX] = {0};
snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP); int32_t nBytes = snprintf(folder, sizeof(folder), "%s%sdnode", tsDataDir, TD_DIRSEP);
snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); if (nBytes <= 0 || nBytes >= PATH_MAX) {
snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); return TSDB_CODE_OUT_OF_BUFFER;
snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE); }
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
terrno = 0; nBytes = snprintf(encryptFile, sizeof(realEncryptFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(realEncryptFile, sizeof(realEncryptFile), "%s%s%s", folder, TD_DIRSEP, DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
nBytes = snprintf(checkFile, sizeof(checkFile), "%s%s%s.bak", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
snprintf(realCheckFile, sizeof(realCheckFile), "%s%s%s", folder, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= PATH_MAX) {
return TSDB_CODE_OUT_OF_BUFFER;
}
if (taosMkDir(folder) != 0) { if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
encryptError("failed to create dir:%s since %s", folder, terrstr()); encryptError("failed to create dir:%s since %s", folder, tstrerror(code));
goto _OVER; goto _OVER;
} }
if(taosCheckExistFile(realCheckFile)){ if (taosCheckExistFile(realCheckFile)) {
if(dmCompareEncryptKey(realCheckFile, key, toLogFile) != 0){ if ((code = dmCompareEncryptKey(realCheckFile, key, toLogFile)) != 0) {
goto _OVER; goto _OVER;
} }
} }
if (!(machineId = tGetMachineId())) { if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (generateEncryptCode(key, machineId, &encryptCode) != 0) { if ((code = generateEncryptCode(key, machineId, &encryptCode)) != 0) {
goto _OVER; goto _OVER;
} }
if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){ if ((code = dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile)) != 0) {
goto _OVER; goto _OVER;
} }
if(dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile) != 0){ if ((code = dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile)) != 0) {
goto _OVER; goto _OVER;
} }
@ -363,8 +452,7 @@ _OVER:
taosMemoryFree(encryptCode); taosMemoryFree(encryptCode);
taosMemoryFree(machineId); taosMemoryFree(machineId);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); encryptError("failed to update encrypt key since %s", tstrerror(code));
encryptError("failed to update encrypt key since %s", terrstr());
} }
return code; return code;
#else #else
@ -374,89 +462,102 @@ _OVER:
extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key); extern int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);
static int32_t dmReadEncryptCodeFile(char* file, char** output){ static int32_t dmReadEncryptCodeFile(char *file, char **output) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
int32_t code = -1; int32_t code = -1;
char *content = NULL; char *content = NULL;
pFile = taosOpenFile(file, TD_FILE_READ); pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to open dnode file:%s since %s", file, terrstr()); dError("failed to open dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
int64_t size = 0; int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to fstat dnode file:%s since %s", file, terrstr()); dError("failed to fstat dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content = taosMemoryMalloc(size + 1); content = taosMemoryMalloc(size + 1);
if (content == NULL) { if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if (taosReadFile(pFile, content, size) != size) { if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
dError("failed to read dnode file:%s since %s", file, terrstr()); dError("failed to read dnode file:%s since %s", file, tstrerror(code));
goto _OVER; goto _OVER;
} }
content[size] = '\0'; content[size] = '\0';
*output = content; *output = content;
content = NULL;
dInfo("succeed to read encryptCode file:%s", file); dInfo("succeed to read encryptCode file:%s", file);
code = 0; code = 0;
_OVER: _OVER:
if(pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
taosMemoryFree(content);
return code; return code;
} }
int32_t dmGetEncryptKey(){ int32_t dmGetEncryptKey() {
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
int32_t code = -1; int32_t code = -1;
char encryptFile[PATH_MAX] = {0}; char encryptFile[PATH_MAX] = {0};
char checkFile[PATH_MAX] = {0}; char checkFile[PATH_MAX] = {0};
char *machineId = NULL; char *machineId = NULL;
char *encryptKey = NULL; char *encryptKey = NULL;
char *content = NULL; char *content = NULL;
snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_ENCRYPT_CODE_FILE); int32_t nBytes = snprintf(encryptFile, sizeof(encryptFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP,
snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE); DM_ENCRYPT_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(encryptFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if(!taosCheckExistFile(encryptFile)){ nBytes = snprintf(checkFile, sizeof(checkFile), "%s%sdnode%s%s", tsDataDir, TD_DIRSEP, TD_DIRSEP, DM_CHECK_CODE_FILE);
if (nBytes <= 0 || nBytes >= sizeof(checkFile)) {
code = TSDB_CODE_OUT_OF_BUFFER;
return code;
}
if (!taosCheckExistFile(encryptFile)) {
code = TSDB_CODE_DNODE_INVALID_ENCRYPT_CONFIG;
dInfo("no exist, checkCode file:%s", encryptFile); dInfo("no exist, checkCode file:%s", encryptFile);
return 0; return 0;
} }
if(dmReadEncryptCodeFile(encryptFile, &content) != 0){ if ((code = dmReadEncryptCodeFile(encryptFile, &content)) != 0) {
goto _OVER; goto _OVER;
} }
if (!(machineId = tGetMachineId())) { if (!(machineId = tGetMachineId())) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
if(checkAndGetCryptKey(content, machineId, &encryptKey) != 0){ if ((code = checkAndGetCryptKey(content, machineId, &encryptKey)) != 0) {
goto _OVER; goto _OVER;
} }
taosMemoryFreeClear(machineId); taosMemoryFreeClear(machineId);
taosMemoryFreeClear(content); taosMemoryFreeClear(content);
if(encryptKey[0] == '\0'){ if (encryptKey[0] == '\0') {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
dError("failed to read key since %s", terrstr()); dError("failed to read key since %s", tstrerror(code));
goto _OVER; goto _OVER;
} }
if(dmCompareEncryptKey(checkFile, encryptKey, true) != 0){ if ((code = dmCompareEncryptKey(checkFile, encryptKey, true)) != 0) {
goto _OVER; goto _OVER;
} }
@ -471,8 +572,7 @@ _OVER:
if (encryptKey != NULL) taosMemoryFree(encryptKey); if (encryptKey != NULL) taosMemoryFree(encryptKey);
if (machineId != NULL) taosMemoryFree(machineId); if (machineId != NULL) taosMemoryFree(machineId);
if (code != 0) { if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to get encrypt key since %s", tstrerror(code));
dError("failed to get encrypt key since %s", terrstr());
} }
return code; return code;
#else #else

77
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -96,57 +96,58 @@ typedef struct STaskChkptInfo {
int8_t dropHTask; int8_t dropHTask;
}STaskChkptInfo; }STaskChkptInfo;
int32_t mndInitStream(SMnode *pMnode); int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode);
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream);
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream); int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId); int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId);
int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt); int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt);
bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock); bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock);
int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId); int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams); int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream); int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode, int32_t acceptCode); int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg); int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg, STrans **pTrans1);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream);
int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
int32_t mndProcessStreamHb(SRpcMsg *pReq); int32_t mndProcessStreamHb(SRpcMsg *pReq);
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
int32_t extractStreamNodeList(SMnode *pMnode); int32_t extractStreamNodeList(SMnode *pMnode);
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated); int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated);
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq); int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId, int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId,
int64_t ts); int64_t ts);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter);
void destroyStreamTaskIter(SStreamTaskIter *pIter); void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); int32_t streamTaskIterGetCurrent(SStreamTaskIter *pIter, SStreamTask **pTask);
void mndInitExecInfo(); int32_t mndInitExecInfo();
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo); void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks); int32_t mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo);
void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo); void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo);
void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo); void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo);
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId); int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId);

690
source/dnode/mnode/impl/src/mndDb.c
View File

File diff suppressed because it is too large Load Diff

267
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -93,6 +93,7 @@ static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName,
} }
int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -110,18 +111,20 @@ int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStb
if (pReq == NULL) { if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
STransAction action = {0}; STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pReq; action.pCont = pReq;
action.contLen = contLen; action.contLen = contLen;
action.msgType = TDMT_VND_CREATE_INDEX; action.msgType = TDMT_VND_CREATE_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return -1; TAOS_RETURN(code);
} }
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -164,6 +167,7 @@ _err:
return NULL; return NULL;
} }
int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -182,6 +186,8 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
if (pReq == NULL) { if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1; return -1;
} }
STransAction action = {0}; STransAction action = {0};
@ -189,7 +195,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
action.pCont = pReq; action.pCont = pReq;
action.contLen = len; action.contLen = len;
action.msgType = TDMT_VND_DROP_INDEX; action.msgType = TDMT_VND_DROP_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
@ -198,7 +204,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
return 0; TAOS_RETURN(code);
} }
void mndCleanupIdx(SMnode *pMnode) { void mndCleanupIdx(SMnode *pMnode) {
@ -332,61 +338,91 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
} }
int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) {
sdbFreeRaw(pRedoRaw); sdbFreeRaw(pRedoRaw);
return -1; return -1;
} }
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { code = TSDB_CODE_MND_RETURN_VALUE_NULL;
sdbFreeRaw(pCommitRaw); if (terrno != 0) code = terrno;
return -1; TAOS_RETURN(code);
} }
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
sdbFreeRaw(pCommitRaw);
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
// static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { // static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
@ -427,10 +463,7 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL; SDbObj *pDb = NULL;
SCreateTagIndexReq createReq = {0}; SCreateTagIndexReq createReq = {0};
if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("idx:%s start to create", createReq.idxName); mInfo("idx:%s start to create", createReq.idxName);
// if (mndCheckCreateIdxReq(&createReq) != 0) { // if (mndCheckCreateIdxReq(&createReq) != 0) {
@ -439,29 +472,30 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByStb(pMnode, createReq.stbName); pDb = mndAcquireDbByStb(pMnode, createReq.stbName);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
pStb = mndAcquireStb(pMnode, createReq.stbName); pStb = mndAcquireStb(pMnode, createReq.stbName);
if (pStb == NULL) { if (pStb == NULL) {
mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName); mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName);
code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
SSIdx idx = {0}; SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx) == 0) { if ((code = mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx; pIdx = idx.pIdx;
} else { } else {
goto _OVER; goto _OVER;
} }
if (pIdx != NULL) { if (pIdx != NULL) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
goto _OVER;
}
code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb); code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb);
if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) { if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) {
@ -472,32 +506,42 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stb:%s, failed to create since %s", createReq.idxName, terrstr()); mError("stb:%s, failed to create since %s", createReq.idxName, tstrerror(code));
} }
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);
mndReleaseIdx(pMnode, pIdx); mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
return code; TAOS_RETURN(code);
} }
int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
return 0; TAOS_RETURN(code);
} }
static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) { static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) {
@ -592,6 +636,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName, static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName,
int on) { int on) {
int32_t code = 0;
taosRLockLatch(&pOld->lock); taosRLockLatch(&pOld->lock);
memcpy(pNew, pOld, sizeof(SStbObj)); memcpy(pNew, pOld, sizeof(SStbObj));
taosRUnLockLatch(&pOld->lock); taosRUnLockLatch(&pOld->lock);
@ -604,28 +649,24 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
int8_t hasIdx = 0; int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx); int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx);
if (tag < 0) { if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST; code = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
col_id_t colId = pOld->pTags[tag].colId; col_id_t colId = pOld->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId));
return -1; TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew));
}
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;
}
SSchema *pTag = pNew->pTags + tag; SSchema *pTag = pNew->pTags + tag;
if (on == 1) { if (on == 1) {
if (hasIdx && tag != 0) { if (hasIdx && tag != 0) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} else { } else {
SSCHMEA_SET_IDX_ON(pTag); SSCHMEA_SET_IDX_ON(pTag);
} }
} else { } else {
if (hasIdx == 0) { if (hasIdx == 0) {
terrno = TSDB_CODE_MND_SMA_NOT_EXIST; code = TSDB_CODE_MND_SMA_NOT_EXIST;
} else { } else {
SSCHMEA_SET_IDX_OFF(pTag); SSCHMEA_SET_IDX_OFF(pTag);
pTag->flags = 0; pTag->flags = 0;
@ -634,11 +675,15 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->tagVer++; pNew->tagVer++;
SSdbRaw *pCommitRaw = mndStbActionEncode(pNew); SSdbRaw *pCommitRaw = mndStbActionEncode(pNew);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
// impl later // impl later
@ -649,17 +694,17 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
// mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); // mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name);
mndTransSetDbName(pTrans, pDb->name, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1), NULL, _OVER);
if (mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -671,7 +716,7 @@ _OVER:
taosMemoryFree(newStb.pCmpr); taosMemoryFree(newStb.pCmpr);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) { int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) {
// build index on first tag, and no index name; // build index on first tag, and no index name;
@ -725,25 +770,23 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
int8_t hasIdx = 0; int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx); int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx);
if (tag < 0) { if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST; code = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
int8_t exist = 0; int8_t exist = 0;
if (tag == 0 && hasIdx == 1) { if (tag == 0 && hasIdx == 1) {
exist = mndCheckIndexNameByTagName(pMnode, &idxObj); exist = mndCheckIndexNameByTagName(pMnode, &idxObj);
if (exist) { if (exist) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
} else if (hasIdx == 1) { } else if (hasIdx == 1) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
col_id_t colId = pStb->pTags[tag].colId; col_id_t colId = pStb->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId) != 0) { TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId));
return -1;
}
// SSchema *pTag = pStb->pTags + tag; // SSchema *pTag = pStb->pTags + tag;
// if (IS_IDX_ON(pTag)) { // if (IS_IDX_ON(pTag)) {
@ -752,7 +795,7 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
// } // }
code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj); code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj);
return code; TAOS_RETURN(code);
} }
static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) { static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) {
@ -763,22 +806,30 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
SStbObj newObj = {0}; SStbObj newObj = {0};
pStb = mndAcquireStb(pMnode, pIdx->stb); pStb = mndAcquireStb(pMnode, pIdx->stb);
if (pStb == NULL) goto _OVER; if (pStb == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name);
mndTransSetDbName(pTrans, pDb->name, NULL); mndTransSetDbName(pTrans, pDb->name, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0), NULL, _OVER);
if (mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -789,7 +840,7 @@ _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);
return code; TAOS_RETURN(code);
} }
int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) { int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -798,13 +849,10 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SIdxObj *pIdx = NULL; SIdxObj *pIdx = NULL;
SDropTagIndexReq req = {0}; SDropTagIndexReq req = {0};
if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req) != 0) { TAOS_CHECK_GOTO(tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("idx:%s, start to drop", req.name); mInfo("idx:%s, start to drop", req.name);
SSIdx idx = {0}; SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx) == 0) { if ((code = mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx; pIdx = idx.pIdx;
} else { } else {
goto _OVER; goto _OVER;
@ -815,7 +863,7 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
code = 0; code = 0;
goto _OVER; goto _OVER;
} else { } else {
terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST; code = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST;
goto _OVER; goto _OVER;
} }
} }
@ -823,23 +871,22 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByIdx(pMnode, req.name); pDb = mndAcquireDbByIdx(pMnode, req.name);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
goto _OVER;
}
code = mndDropIdx(pMnode, pReq, pDb, pIdx); code = mndDropIdx(pMnode, pReq, pDb, pIdx);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("idx:%s, failed to drop since %s", req.name, terrstr()); mError("idx:%s, failed to drop since %s", req.name, tstrerror(code));
} }
mndReleaseIdx(pMnode, pIdx); mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) { static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
// do nothing // do nothing
@ -847,6 +894,7 @@ static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
} }
int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
@ -856,17 +904,17 @@ int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
if (pIter == NULL) break; if (pIter == NULL) break;
if (pIdx->stbUid == pStb->uid) { if (pIdx->stbUid == pStb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
return -1; TAOS_RETURN(code);
} }
} }
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) { int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) {
@ -891,6 +939,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb
return -1; return -1;
} }
int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
@ -900,15 +949,15 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
if (pIter == NULL) break; if (pIter == NULL) break;
if (pIdx->dbUid == pDb->uid) { if (pIdx->dbUid == pDb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; TAOS_RETURN(code);
} }
} }
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
} }
return 0; TAOS_RETURN(code);
} }

12
source/dnode/mnode/impl/src/mndIndexCom.c
View File

@ -30,7 +30,7 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA); SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA);
SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX); SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX);
terrno = 0; int32_t code = 0;
if (pSma == NULL && pIdx == NULL) return 0; if (pSma == NULL && pIdx == NULL) return 0;
@ -40,8 +40,8 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pSma; idx->pIdx = pSma;
} else { } else {
mndReleaseSma(pMnode, pSma); mndReleaseSma(pMnode, pSma);
terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; code = TSDB_CODE_MND_SMA_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
} else { } else {
if (type == SDB_IDX) { if (type == SDB_IDX) {
@ -49,9 +49,9 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pIdx; idx->pIdx = pIdx;
} else { } else {
mndReleaseIdx(pMnode, pIdx); mndReleaseIdx(pMnode, pIdx);
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
} }
return 0; TAOS_RETURN(code);
} }

55
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -18,10 +18,11 @@
#include "systable.h" #include "systable.h"
static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) { if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
@ -35,10 +36,11 @@ static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t co
} }
*pDst = schema; *pDst = schema;
return 0; TAOS_RETURN(code);
} }
static int32_t mndInsInitMeta(SHashObj *hash) { static int32_t mndInsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0}; STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName)); tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -55,24 +57,23 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
meta.numOfColumns = pInfosTableMeta[i].colNum; meta.numOfColumns = pInfosTableMeta[i].colNum;
meta.sysInfo = pInfosTableMeta[i].sysInfo; meta.sysInfo = pInfosTableMeta[i].sysInfo;
if (mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)) { TAOS_CHECK_RETURN(mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas));
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo, int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo,
STableMetaRsp *pRsp) { STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) { if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *pMeta = NULL; STableMetaRsp *pMeta = NULL;
@ -84,40 +85,41 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
if (!sysinfo && pMeta->sysInfo) { if (!sysinfo && pMeta->sysInfo) {
mError("no permission to get schema of table name:%s", tbName); mError("no permission to get schema of table name:%s", tbName);
terrno = TSDB_CODE_PAR_PERMISSION_DENIED; code = TSDB_CODE_PAR_PERMISSION_DENIED;
return -1; TAOS_RETURN(code);
} }
*pRsp = *pMeta; *pRsp = *pMeta;
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0; TAOS_RETURN(code);
} }
int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) { if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
strcpy(pRsp->tbName, pMeta->tbName); strcpy(pRsp->tbName, pMeta->tbName);
@ -129,22 +131,21 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt)); pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt));
return 0; TAOS_RETURN(code);
} }
int32_t mndInitInfos(SMnode *pMnode) { int32_t mndInitInfos(SMnode *pMnode) {
pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pMnode->infosMeta == NULL) { if (pMnode->infosMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
return mndInsInitMeta(pMnode->infosMeta); return mndInsInitMeta(pMnode->infosMeta);

212
source/dnode/mnode/impl/src/mndMain.c
View File

@ -50,8 +50,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0; int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock); taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) { if (pMnode->stopped) {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
code = -1;
} else if (!mndIsLeader(pMnode)) { } else if (!mndIsLeader(pMnode)) {
code = -1; code = -1;
} else { } else {
@ -63,7 +62,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
#endif #endif
} }
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
return code; TAOS_RETURN(code);
} }
static inline void mndReleaseRpc(SMnode *pMnode) { static inline void mndReleaseRpc(SMnode *pMnode) {
@ -78,6 +77,7 @@ static inline void mndReleaseRpc(SMnode *pMnode) {
} }
static void *mndBuildTimerMsg(int32_t *pContLen) { static void *mndBuildTimerMsg(int32_t *pContLen) {
terrno = 0;
SMTimerReq timerReq = {0}; SMTimerReq timerReq = {0};
int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq); int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq);
@ -329,6 +329,7 @@ static int32_t minCronTime() {
return min <= 1 ? 2 : min; return min <= 1 ? 2 : min;
} }
void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
int32_t code = 0;
if (sec % tsTtlPushIntervalSec == 0) { if (sec % tsTtlPushIntervalSec == 0) {
mndPullupTtl(pMnode); mndPullupTtl(pMnode);
} }
@ -378,14 +379,14 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
} }
if (sec % (tsArbHeartBeatIntervalSec) == 0) { if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) { if ((code = mndPullupArbHeartbeat(pMnode)) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr()); mError("failed to pullup arb heartbeat, since:%s", tstrerror(code));
} }
} }
if (sec % (tsArbCheckSyncIntervalSec) == 0) { if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) { if ((code = mndPullupArbCheckSync(pMnode)) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr()); mError("failed to pullup arb check sync, since:%s", tstrerror(code));
} }
} }
} }
@ -426,17 +427,18 @@ static void *mndThreadFp(void *param) {
} }
static int32_t mndInitTimer(SMnode *pMnode) { static int32_t mndInitTimer(SMnode *pMnode) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode) != 0) { if ((code = taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode)) != 0) {
mError("failed to create timer thread since %s", strerror(errno)); mError("failed to create timer thread since %s", strerror(errno));
return -1; TAOS_RETURN(code);
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("mnode-timer", "initialized"); tmsgReportStartup("mnode-timer", "initialized");
return 0; TAOS_RETURN(code);
} }
static void mndCleanupTimer(SMnode *pMnode) { static void mndCleanupTimer(SMnode *pMnode) {
@ -447,41 +449,41 @@ static void mndCleanupTimer(SMnode *pMnode) {
} }
static int32_t mndCreateDir(SMnode *pMnode, const char *path) { static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
int32_t code = 0;
pMnode->path = taosStrdup(path); pMnode->path = taosStrdup(path);
if (pMnode->path == NULL) { if (pMnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
if (taosMkDir(pMnode->path) != 0) { if (taosMkDir(pMnode->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndInitWal(SMnode *pMnode) { static int32_t mndInitWal(SMnode *pMnode) {
int32_t code = 0;
char path[PATH_MAX + 20] = {0}; char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = { SWalCfg cfg = {.vgId = 1,
.vgId = 1, .fsyncPeriod = 0,
.fsyncPeriod = 0, .rollPeriod = -1,
.rollPeriod = -1, .segSize = -1,
.segSize = -1, .retentionPeriod = 0,
.retentionPeriod = 0, .retentionSize = 0,
.retentionSize = 0, .level = TAOS_WAL_FSYNC,
.level = TAOS_WAL_FSYNC, .encryptAlgorithm = 0,
.encryptAlgorithm = 0, .encryptKey = {0}};
.encryptKey = {0}
};
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){ if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0; cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){ if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1; TAOS_RETURN(code);
} }
else{ else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
@ -491,11 +493,13 @@ static int32_t mndInitWal(SMnode *pMnode) {
pMnode->pWal = walOpen(path, &cfg); pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) { if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path); code = TSDB_CODE_MND_RETURN_VALUE_NULL;
return -1; if (terrno != 0) code = terrno;
mError("failed to open wal since %s. wal:%s", tstrerror(code), path);
TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static void mndCloseWal(SMnode *pMnode) { static void mndCloseWal(SMnode *pMnode) {
@ -506,6 +510,7 @@ static void mndCloseWal(SMnode *pMnode) {
} }
static int32_t mndInitSdb(SMnode *pMnode) { static int32_t mndInitSdb(SMnode *pMnode) {
int32_t code = 0;
SSdbOpt opt = {0}; SSdbOpt opt = {0};
opt.path = pMnode->path; opt.path = pMnode->path;
opt.pMnode = pMnode; opt.pMnode = pMnode;
@ -513,10 +518,12 @@ static int32_t mndInitSdb(SMnode *pMnode) {
pMnode->pSdb = sdbInit(&opt); pMnode->pSdb = sdbInit(&opt);
if (pMnode->pSdb == NULL) { if (pMnode->pSdb == NULL) {
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndOpenSdb(SMnode *pMnode) { static int32_t mndOpenSdb(SMnode *pMnode) {
@ -542,48 +549,47 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle
step.initFp = initFp; step.initFp = initFp;
step.cleanupFp = cleanupFp; step.cleanupFp = cleanupFp;
if (taosArrayPush(pMnode->pSteps, &step) == NULL) { if (taosArrayPush(pMnode->pSteps, &step) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
return 0; TAOS_RETURN(0);
} }
static int32_t mndInitSteps(SMnode *pMnode) { static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal));
if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb));
if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans));
if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster));
if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode));
if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode));
if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode));
if (mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup));
if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode));
if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser));
if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant));
if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege));
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct));
if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream));
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic));
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer));
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe));
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup));
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb));
if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma));
if (mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx));
if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos));
if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs));
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb));
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc));
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView));
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact));
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail));
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL));
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile));
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow));
if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery));
if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync));
if (mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem));
return 0; return 0;
} }
@ -609,17 +615,16 @@ static void mndCleanupSteps(SMnode *pMnode, int32_t pos) {
} }
static int32_t mndExecSteps(SMnode *pMnode) { static int32_t mndExecSteps(SMnode *pMnode) {
int32_t code = 0;
int32_t size = taosArrayGetSize(pMnode->pSteps); int32_t size = taosArrayGetSize(pMnode->pSteps);
for (int32_t pos = 0; pos < size; pos++) { for (int32_t pos = 0; pos < size; pos++) {
SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, pos); SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, pos);
if (pStep->initFp == NULL) continue; if (pStep->initFp == NULL) continue;
if ((*pStep->initFp)(pMnode) != 0) { if ((code = (*pStep->initFp)(pMnode)) != 0) {
int32_t code = terrno; mError("%s exec failed since %s, start to cleanup", pStep->name, tstrerror(code));
mError("%s exec failed since %s, start to cleanup", pStep->name, terrstr());
mndCleanupSteps(pMnode, pos); mndCleanupSteps(pMnode, pos);
terrno = code; TAOS_RETURN(code);
return -1;
} else { } else {
mInfo("%s is initialized", pStep->name); mInfo("%s is initialized", pStep->name);
tmsgReportStartup(pStep->name, "initialized"); tmsgReportStartup(pStep->name, "initialized");
@ -627,7 +632,7 @@ static int32_t mndExecSteps(SMnode *pMnode) {
} }
pMnode->clusterId = mndGetClusterId(pMnode); pMnode->clusterId = mndGetClusterId(pMnode);
return 0; TAOS_RETURN(0);
} }
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
@ -642,6 +647,7 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
} }
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
terrno = 0;
mInfo("start to open mnode in %s", path); mInfo("start to open mnode in %s", path);
SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode)); SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode));
@ -668,7 +674,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
int32_t code = mndCreateDir(pMnode, path); int32_t code = mndCreateDir(pMnode, path);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode); mndClose(pMnode);
terrno = code; terrno = code;
return NULL; return NULL;
@ -677,7 +683,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndInitSteps(pMnode); code = mndInitSteps(pMnode);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode); mndClose(pMnode);
terrno = code; terrno = code;
return NULL; return NULL;
@ -686,7 +692,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndExecSteps(pMnode); code = mndExecSteps(pMnode);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode); mndClose(pMnode);
terrno = code; terrno = code;
return NULL; return NULL;
@ -762,45 +768,47 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
int32_t code = syncProcessMsg(pMgmt->sync, pMsg); int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
if (code != 0) { if (code != 0) {
mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType), mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
terrstr(), code); tstrerror(code), code);
} }
return code; return code;
} }
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0; int32_t code = 0;
if (!IsReq(pMsg)) TAOS_RETURN(code);
if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY || if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY ||
pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT || pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT ||
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK || pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK ||
pMsg->msgType == TDMT_SCH_TASK_NOTIFY) { pMsg->msgType == TDMT_SCH_TASK_NOTIFY) {
return 0; TAOS_RETURN(code);
} }
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
taosThreadRwlockRdlock(&pMnode->lock); taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) { if (pMnode->stopped) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
return -1; TAOS_RETURN(code);
} }
terrno = 0; terrno = 0;
SSyncState state = syncGetState(pMnode->syncMgmt.sync); SSyncState state = syncGetState(pMnode->syncMgmt.sync);
if (terrno != 0) { if (terrno != 0) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
return -1; code = terrno;
TAOS_RETURN(code);
} }
if (state.state != TAOS_SYNC_STATE_LEADER) { if (state.state != TAOS_SYNC_STATE_LEADER) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_NOT_LEADER; code = TSDB_CODE_SYN_NOT_LEADER;
goto _OVER; goto _OVER;
} }
if (!state.restored || !pMnode->restored) { if (!state.restored || !pMnode->restored) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_RESTORING; code = TSDB_CODE_SYN_RESTORING;
goto _OVER; goto _OVER;
} }
@ -812,7 +820,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
#endif #endif
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
return 0; TAOS_RETURN(code);
_OVER: _OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
@ -824,19 +832,17 @@ _OVER:
pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state)); pMnode->stopped, state.restored, syncStr(state.state));
return -1; TAOS_RETURN(code);
} }
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SEpSet epSet = {0}; SEpSet epSet = {0};
int32_t tmpCode = terrno;
mndGetMnodeEpSet(pMnode, &epSet); mndGetMnodeEpSet(pMnode, &epSet);
terrno = tmpCode;
mGDebug( mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "
"role:%s, redirect numOfEps:%d inUse:%d, type:%s", "role:%s, redirect numOfEps:%d inUse:%d, type:%s",
pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored, pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code), pMnode->restored, pMnode->stopped, state.restored,
syncStr(state.state), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType)); syncStr(state.state), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType));
if (epSet.numOfEps <= 0) return -1; if (epSet.numOfEps <= 0) return -1;
@ -853,10 +859,10 @@ _OVER:
pMsg->info.rspLen = contLen; pMsg->info.rspLen = contLen;
} }
return -1; TAOS_RETURN(code);
} }
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) { int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -867,12 +873,12 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
fpExt = pMnode->msgFpExt[TMSG_INDEX(pMsg->msgType)]; fpExt = pMnode->msgFpExt[TMSG_INDEX(pMsg->msgType)];
if (fpExt == NULL) { if (fpExt == NULL) {
mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
return -1; TAOS_RETURN(code);
} }
} }
if (mndCheckMnodeState(pMsg) != 0) return -1; TAOS_CHECK_RETURN(mndCheckMnodeState(pMsg));
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
if (fp) if (fp)
@ -886,6 +892,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
} else if (code == 0) { } else if (code == 0) {
mGTrace("msg:%p, successfully processed", pMsg); mGTrace("msg:%p, successfully processed", pMsg);
} else { } else {
//TODO removve this wrong set code
if (code == -1) { if (code == -1) {
code = terrno; code = terrno;
} }
@ -893,7 +900,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo* pQueueInfo) {
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
} }
return code; TAOS_RETURN(code);
} }
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
@ -925,7 +932,8 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpc(pMnode) != 0) return -1; int32_t code = 0;
TAOS_CHECK_RETURN(mndAcquireRpc(pMnode));
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs(); int64_t ms = taosGetTimestampMs();
@ -937,7 +945,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) { pStbInfo->stbs == NULL) {
mndReleaseRpc(pMnode); mndReleaseRpc(pMnode);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
// cluster info // cluster info
@ -1063,7 +1073,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
} }
mndReleaseRpc(pMnode); mndReleaseRpc(pMnode);
return 0; TAOS_RETURN(code);
} }
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {

256
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -69,6 +69,7 @@ int32_t mndInitMnode(SMnode *pMnode) {
void mndCleanupMnode(SMnode *pMnode) {} void mndCleanupMnode(SMnode *pMnode) {}
SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) { SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
terrno = 0;
SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId); SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId);
if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
@ -82,13 +83,18 @@ void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
} }
static int32_t mndCreateDefaultMnode(SMnode *pMnode) { static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
int32_t code = 0;
SMnodeObj mnodeObj = {0}; SMnodeObj mnodeObj = {0};
mnodeObj.id = 1; mnodeObj.id = 1;
mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.createdTime = taosGetTimestampMs();
mnodeObj.updateTime = mnodeObj.createdTime; mnodeObj.updateTime = mnodeObj.createdTime;
SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw); mInfo("mnode:%d, will be created when deploying, raw:%p", mnodeObj.id, pRaw);
@ -97,25 +103,27 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1; return -1;
} }
mInfo("trans:%d, used to create mnode:%d", pTrans->id, mnodeObj.id); mInfo("trans:%d, used to create mnode:%d", pTrans->id, mnodeObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; TAOS_RETURN(code);
} }
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) {
@ -188,16 +196,19 @@ _OVER:
} }
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
int32_t code = 0;
mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj); mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id); pObj->pDnode = sdbAcquireNotReadyObj(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) { if (pObj->pDnode == NULL) {
mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
int32_t code = 0;
} }
pObj->syncState = TAOS_SYNC_STATE_OFFLINE; pObj->syncState = TAOS_SYNC_STATE_OFFLINE;
mndReloadSyncConfig(pSdb->pMnode); mndReloadSyncConfig(pSdb->pMnode);
return 0; TAOS_RETURN(code);
} }
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
@ -271,38 +282,59 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
} }
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
int32_t mndSetRestoreCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { int32_t mndSetRestoreCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) { static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pCreateReq, SEpSet *pCreateEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pCreateReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq); tSerializeSDCreateMnodeReq(pReq, contLen, pCreateReq);
@ -315,15 +347,16 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p
.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED, .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq, static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeTypeReq *pAlterMnodeTypeReq,
SEpSet *pAlterMnodeTypeEpSet) { SEpSet *pAlterMnodeTypeEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq);
@ -337,14 +370,15 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, SDAlterMnodeType
.acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER, .acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq); tSerializeSDCreateMnodeReq(pReq, contLen, pAlterReq);
@ -357,15 +391,16 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA
.acceptableCode = 0, .acceptableCode = 0,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) {
int32_t code = 0;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq);
@ -378,11 +413,11 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop
.acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED, .acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED,
}; };
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -426,9 +461,9 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -474,9 +509,9 @@ int32_t mndSetRestoreCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildCreateMnodeRedoAction(pTrans, &createReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -517,9 +552,9 @@ static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
@ -565,19 +600,23 @@ int32_t mndSetRestoreAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, S
createEpset.eps[0].port = pDnode->port; createEpset.eps[0].port = pDnode->port;
memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; TAOS_CHECK_RETURN(mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset));
return 0; TAOS_RETURN(0);
} }
static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
SMnodeObj mnodeObj = {0}; SMnodeObj mnodeObj = {0};
mnodeObj.id = pDnode->id; mnodeObj.id = pDnode->id;
@ -586,8 +625,8 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeObj.role = TAOS_SYNC_ROLE_LEARNER; mnodeObj.role = TAOS_SYNC_ROLE_LEARNER;
mnodeObj.lastIndex = pMnode->applied; mnodeObj.lastIndex = pMnode->applied;
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj), NULL, _OVER);
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj), NULL, _OVER);
SMnodeObj mnodeLeaderObj = {0}; SMnodeObj mnodeLeaderObj = {0};
mnodeLeaderObj.id = pDnode->id; mnodeLeaderObj.id = pDnode->id;
@ -596,15 +635,15 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER; mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER;
mnodeLeaderObj.lastIndex = pMnode->applied + 1; mnodeLeaderObj.lastIndex = pMnode->applied + 1;
if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj), NULL, _OVER);
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
@ -614,19 +653,14 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateMnodeReq createReq = {0}; SMCreateMnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("mnode:%d, start to create", createReq.dnodeId); mInfo("mnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE), NULL, _OVER);
goto _OVER;
}
pObj = mndAcquireMnode(pMnode, createReq.dnodeId); pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
if (pObj != NULL) { if (pObj != NULL) {
terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; code = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) {
goto _OVER; goto _OVER;
@ -634,17 +668,17 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
terrno = TSDB_CODE_MND_TOO_MANY_MNODES; code = TSDB_CODE_MND_TOO_MANY_MNODES;
goto _OVER; goto _OVER;
} }
if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE; code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER; goto _OVER;
} }
@ -665,27 +699,38 @@ _OVER:
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj, static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj,
bool force) { bool force) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SDDropMnodeReq dropReq = {0}; SDDropMnodeReq dropReq = {0};
@ -700,32 +745,32 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
if (totalMnodes == 2) { if (totalMnodes == 2) {
if (force) { if (force) {
mError("cant't force drop dnode, since a mnode on it and replica is 2"); mError("cant't force drop dnode, since a mnode on it and replica is 2");
terrno = TSDB_CODE_MNODE_ONLY_TWO_MNODE; code = TSDB_CODE_MNODE_ONLY_TWO_MNODE;
return -1; TAOS_RETURN(code);
} }
mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes);
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
if (!force) { if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
} }
} else if (totalMnodes == 3) { } else if (totalMnodes == 3) {
mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes); mInfo("vgId:1, has %d mnodes, exec redo action first", totalMnodes);
if (!force) { if (!force) {
if (mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet) != 0) return -1; TAOS_CHECK_RETURN(mndBuildDropMnodeRedoAction(pTrans, &dropReq, &dropEpSet));
} }
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj));
} else { } else {
return -1; TAOS_RETURN(-1);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) { int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
pObj->lastIndex = pMnode->applied; pObj->lastIndex = pMnode->applied;
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force));
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj));
return 0; return 0;
} }
@ -734,19 +779,23 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) {
STrans *pTrans = NULL; STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-mnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
@ -755,38 +804,35 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
SMnodeObj *pObj = NULL; SMnodeObj *pObj = NULL;
SMDropMnodeReq dropReq = {0}; SMDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("mnode:%d, start to drop", dropReq.dnodeId); mInfo("mnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireMnode(pMnode, dropReq.dnodeId); pObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
if (pMnode->selfDnodeId == dropReq.dnodeId) { if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER; code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER; goto _OVER;
} }
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES; code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER; goto _OVER;
} }
if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) { if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) {
terrno = TSDB_CODE_DNODE_OFFLINE; code = TSDB_CODE_DNODE_OFFLINE;
goto _OVER; goto _OVER;
} }
@ -805,7 +851,7 @@ _OVER:
mndReleaseMnode(pMnode, pObj); mndReleaseMnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
@ -892,13 +938,11 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
#if 1 #if 1
return 0; return 0;
#else #else
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SDAlterMnodeReq alterReq = {0}; SDAlterMnodeReq alterReq = {0};
if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1}; SMnodeOpt option = {.deploy = true, .numOfReplicas = alterReq.replica, .selfIndex = -1};
memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas)); memcpy(option.replicas, alterReq.replicas, sizeof(alterReq.replicas));
@ -913,9 +957,9 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
return 0; return 0;
} }
if (mndWriteFile(pMnode->path, &option) != 0) { if ((code = mndWriteFile(pMnode->path, &option)) != 0) {
mError("failed to write mnode file since %s", terrstr()); mError("failed to write mnode file since %s", terrstr());
return -1; TAOS_RETURN(code);
} }
SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1}; SSyncCfg cfg = {.replicaNum = alterReq.replica, .myIndex = -1};
@ -939,14 +983,14 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) {
} }
} }
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); code = syncReconfig(pMnode->syncMgmt.sync, &cfg);
if (code != 0) { if (code != 0) {
mError("failed to sync reconfig since %s", terrstr()); mError("failed to sync reconfig since %s", terrstr());
} else { } else {
mInfo("alter mnode sync success"); mInfo("alter mnode sync success");
} }
return code; TAOS_RETURN(code);
#endif #endif
} }

53
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -19,10 +19,11 @@
// connection/application/ // connection/application/
int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) { if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
@ -34,10 +35,11 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S
} }
*pDst = schema; *pDst = schema;
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
int32_t mndPerfsInitMeta(SHashObj *hash) { int32_t mndPerfsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0}; STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName)); tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -53,56 +55,56 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName)); tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName));
meta.numOfColumns = pSysDbTableMeta[i].colNum; meta.numOfColumns = pSysDbTableMeta[i].colNum;
if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) { TAOS_CHECK_RETURN(mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas));
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) { if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) { if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
*pRsp = *meta; *pRsp = *meta;
pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema));
return 0; TAOS_RETURN(code);
} }
int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->perfsMeta) { if (NULL == pMnode->perfsMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
strcpy(pRsp->tbName, pMeta->tbName); strcpy(pRsp->tbName, pMeta->tbName);
@ -114,20 +116,21 @@ int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *t
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0; TAOS_RETURN(code);
} }
int32_t mndInitPerfs(SMnode *pMnode) { int32_t mndInitPerfs(SMnode *pMnode) {
int32_t code = 0;
pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (pMnode->perfsMeta == NULL) { if (pMnode->perfsMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
return mndPerfsInitMeta(pMnode->perfsMeta); return mndPerfsInitMeta(pMnode->perfsMeta);

77
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -85,22 +85,23 @@ static void mndCancelGetNextApp(SMnode *pMnode, void *pIter);
static int32_t mndProcessSvrVerReq(SRpcMsg *pReq); static int32_t mndProcessSvrVerReq(SRpcMsg *pReq);
int32_t mndInitProfile(SMnode *pMnode) { int32_t mndInitProfile(SMnode *pMnode) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
// in ms // in ms
int32_t checkTime = tsShellActivityTimer * 2 * 1000; int32_t checkTime = tsShellActivityTimer * 2 * 1000;
pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn"); pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, false, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->connCache == NULL) { if (pMgmt->connCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr()); mError("failed to alloc profile cache since %s", terrstr());
return -1; TAOS_RETURN(code);
} }
pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app"); pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app");
if (pMgmt->appCache == NULL) { if (pMgmt->appCache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc profile cache since %s", terrstr()); mError("failed to alloc profile cache since %s", terrstr());
return -1; TAOS_RETURN(code);
} }
mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq); mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq);
@ -116,7 +117,7 @@ int32_t mndInitProfile(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp);
return 0; TAOS_RETURN(code);
} }
void mndCleanupProfile(SMnode *pMnode) { void mndCleanupProfile(SMnode *pMnode) {
@ -384,6 +385,7 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq)
static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); } static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); }
static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) { static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) {
terrno = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId)); SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId));
@ -431,13 +433,16 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) {
} }
static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) { static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) {
int32_t code = 0;
SAppHbReq *pReq = &pHbReq->app; SAppHbReq *pReq = &pHbReq->app;
SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId); SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId);
if (pApp == NULL) { if (pApp == NULL) {
pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq); pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq);
if (pApp == NULL) { if (pApp == NULL) {
mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr()); mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else { } else {
mDebug("a new app %" PRIx64 " is created", pReq->appId); mDebug("a new app %" PRIx64 " is created", pReq->appId);
mndReleaseApp(pMnode, pApp); mndReleaseApp(pMnode, pApp);
@ -475,6 +480,7 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) {
static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq,
SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) { SClientHbBatchRsp *pBatchRsp, SConnPreparedObj *pObj) {
int32_t code = 0;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
SRpcConnInfo connInfo = pMsg->info.conn; SRpcConnInfo connInfo = pMsg->info.conn;
@ -492,7 +498,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
pHbReq->app.pid, pHbReq->app.name, 0); pHbReq->app.pid, pHbReq->app.name, 0);
if (pConn == NULL) { if (pConn == NULL) {
mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr()); mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} else { } else {
mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id);
} }
@ -501,9 +509,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic));
if (rspBasic == NULL) { if (rspBasic == NULL) {
mndReleaseConn(pMnode, pConn, true); mndReleaseConn(pMnode, pConn, true);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr()); mError("user:%s, conn:%u failed to process hb while since %s", pConn->user, pBasic->connId, terrstr());
return -1; TAOS_RETURN(code);
} }
mndSaveQueryList(pConn, pBasic); mndSaveQueryList(pConn, pBasic);
@ -539,9 +547,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
hbRsp.info = taosArrayInit(kvNum, sizeof(SKv)); hbRsp.info = taosArrayInit(kvNum, sizeof(SKv));
if (NULL == hbRsp.info) { if (NULL == hbRsp.info) {
mError("taosArrayInit %d rsp kv failed", kvNum); mError("taosArrayInit %d rsp kv failed", kvNum);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
tFreeClientHbRsp(&hbRsp); tFreeClientHbRsp(&hbRsp);
return -1; TAOS_RETURN(code);
} }
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
@ -554,8 +562,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer); mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer);
SDynViewVersion *pRspVer = NULL; SDynViewVersion *pRspVer = NULL;
if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) { if (0 != (code = mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer))) {
return -1; TAOS_RETURN(code);
} }
if (needCheck) { if (needCheck) {
@ -647,13 +655,14 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
} }
static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) { static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SClientHbBatchReq batchReq = {0}; SClientHbBatchReq batchReq = {0};
if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) { if (tDeserializeSClientHbBatchReq(pReq->pCont, pReq->contLen, &batchReq) != 0) {
taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; TAOS_RETURN(code);
} }
SConnPreparedObj obj = {0}; SConnPreparedObj obj = {0};
@ -699,31 +708,27 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq) {
taosArrayDestroy(obj.pQnodeList); taosArrayDestroy(obj.pQnodeList);
return 0; TAOS_RETURN(code);
} }
static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillQueryReq killReq = {0}; SKillQueryReq killReq = {0};
if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); mInfo("kill query msg is received, queryId:%s", killReq.queryStrId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY));
return -1;
}
int32_t connId = 0; int32_t connId = 0;
uint64_t queryId = 0; uint64_t queryId = 0;
char *p = strchr(killReq.queryStrId, ':'); char *p = strchr(killReq.queryStrId, ':');
if (NULL == p) { if (NULL == p) {
mError("invalid query id %s", killReq.queryStrId); mError("invalid query id %s", killReq.queryStrId);
terrno = TSDB_CODE_MND_INVALID_QUERY_ID; code = TSDB_CODE_MND_INVALID_QUERY_ID;
return -1; TAOS_RETURN(code);
} }
*p = 0; *p = 0;
connId = taosStr2Int32(killReq.queryStrId, NULL, 16); connId = taosStr2Int32(killReq.queryStrId, NULL, 16);
@ -732,40 +737,36 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t)); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t));
if (pConn == NULL) { if (pConn == NULL) {
mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId); mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID; code = TSDB_CODE_MND_INVALID_CONN_ID;
return -1; TAOS_RETURN(code);
} else { } else {
mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user); mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user);
pConn->killId = queryId; pConn->killId = queryId;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return 0; TAOS_RETURN(code);
} }
} }
static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { static int32_t mndProcessKillConnReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SKillConnReq killReq = {0}; SKillConnReq killReq = {0};
if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { TAOS_CHECK_RETURN(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN));
return -1;
}
SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t));
if (pConn == NULL) { if (pConn == NULL) {
mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); mError("connId:%u, failed to kill connection, conn not exist", killReq.connId);
terrno = TSDB_CODE_MND_INVALID_CONN_ID; code = TSDB_CODE_MND_INVALID_CONN_ID;
return -1; TAOS_RETURN(code);
} else { } else {
mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user); mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user);
pConn->killed = 1; pConn->killed = 1;
taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); taosCacheRelease(pMgmt->connCache, (void **)&pConn, false);
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
} }

190
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -138,15 +138,16 @@ _OVER:
} }
static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) { static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) {
int32_t code = 0;
mTrace("qnode:%d, perform insert action, row:%p", pObj->id, pObj); mTrace("qnode:%d, perform insert action, row:%p", pObj->id, pObj);
pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) { if (pObj->pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) {
@ -166,27 +167,42 @@ static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew
} }
static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) { bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
@ -194,6 +210,7 @@ bool mndQnodeInDnode(SQnodeObj *pQnode, int32_t dnodeId) {
} }
int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDCreateQnodeReq createReq = {0}; SDCreateQnodeReq createReq = {0};
createReq.dnodeId = pDnode->id; createReq.dnodeId = pDnode->id;
@ -212,23 +229,24 @@ int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeOb
action.msgType = TDMT_DND_CREATE_QNODE; action.msgType = TDMT_DND_CREATE_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED; action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0}; SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +257,12 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_QNODE; action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) { if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) {
@ -256,22 +274,26 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
qnodeObj.updateTime = qnodeObj.createdTime; qnodeObj.updateTime = qnodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
if (mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
@ -281,19 +303,14 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateQnodeReq createReq = {0}; SMCreateQnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("qnode:%d, start to create", createReq.dnodeId); mInfo("qnode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE), NULL, _OVER);
goto _OVER;
}
pObj = mndAcquireQnode(pMnode, createReq.dnodeId); pObj = mndAcquireQnode(pMnode, createReq.dnodeId);
if (pObj != NULL) { if (pObj != NULL) {
terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST; code = TSDB_CODE_MND_QNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) {
goto _OVER; goto _OVER;
@ -301,7 +318,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
@ -320,34 +337,47 @@ _OVER:
mndReleaseQnode(pMnode, pObj); mndReleaseQnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
if (mndTransAppendCommitlog(pTrans, pCommitRaw))
;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED))
;
TAOS_RETURN(code);
} }
static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) {
int32_t code = 0;
SDDropQnodeReq dropReq = {0}; SDDropQnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -358,20 +388,20 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
action.msgType = TDMT_DND_DROP_QNODE; action.msgType = TDMT_DND_DROP_QNODE;
action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) { int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropQnodeRedoLogs(pTrans, pObj));
if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropQnodeCommitLogs(pTrans, pObj));
if (!force) { if (!force) {
if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj));
} }
return 0; return 0;
} }
@ -380,18 +410,22 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
@ -400,23 +434,20 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
SQnodeObj *pObj = NULL; SQnodeObj *pObj = NULL;
SMDropQnodeReq dropReq = {0}; SMDropQnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("qnode:%d, start to drop", dropReq.dnodeId); mInfo("qnode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireQnode(pMnode, dropReq.dnodeId); pObj = mndAcquireQnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -435,10 +466,11 @@ _OVER:
mndReleaseQnode(pMnode, pObj); mndReleaseQnode(pMnode, pObj);
tFreeSDDropQnodeReq(&dropReq); tFreeSDDropQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) { int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SQnodeObj *pObj = NULL; SQnodeObj *pObj = NULL;
@ -447,8 +479,8 @@ int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit) {
SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad)); SArray *qnodeList = taosArrayInit(5, sizeof(SQueryNodeLoad));
if (NULL == qnodeList) { if (NULL == qnodeList) {
mError("failed to alloc epSet while process qnode list req"); mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return terrno; TAOS_RETURN(code);
} }
while (1) { while (1) {
@ -484,20 +516,14 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
SQnodeListReq qlistReq = {0}; SQnodeListReq qlistReq = {0};
SQnodeListRsp qlistRsp = {0}; SQnodeListRsp qlistRsp = {0};
if (tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSQnodeListReq(pReq->pCont, pReq->contLen, &qlistReq), NULL, _OVER);
mError("failed to parse qnode list req");
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
if (mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum) != 0) { TAOS_CHECK_GOTO(mndCreateQnodeList(pMnode, &qlistRsp.qnodeList, qlistReq.rowNum), NULL, _OVER);
goto _OVER;
}
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -509,7 +535,7 @@ static int32_t mndProcessQnodeListReq(SRpcMsg *pReq) {
_OVER: _OVER:
tFreeSQnodeListRsp(&qlistRsp); tFreeSQnodeListRsp(&qlistRsp);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

6
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -88,7 +88,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
void *pRsp = NULL; void *pRsp = NULL;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { if ((code = tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("tDeserializeSBatchReq failed"); mError("tDeserializeSBatchReq failed");
goto _exit; goto _exit;
@ -119,7 +119,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) {
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req->msgType)]; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req->msgType)];
if (fp == NULL) { if (fp == NULL) {
mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
taosArrayDestroy(batchRsp.pRsps); taosArrayDestroy(batchRsp.pRsps);
return -1; return -1;
} }
@ -164,7 +164,7 @@ _exit:
taosArrayDestroyEx(batchReq.pMsgs, tFreeSBatchReqMsg); taosArrayDestroyEx(batchReq.pMsgs, tFreeSBatchReqMsg);
taosArrayDestroyEx(batchRsp.pRsps, mnodeFreeSBatchRspMsg); taosArrayDestroyEx(batchRsp.pRsps, mnodeFreeSBatchRspMsg);
return code; TAOS_RETURN(code);
} }
int32_t mndInitQuery(SMnode *pMnode) { int32_t mndInitQuery(SMnode *pMnode) {

82
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -50,17 +50,17 @@ static bool isCountWindowStreamTask(SSubplan* pPlan) {
int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
int64_t watermark, int64_t deleteMark) { int64_t watermark, int64_t deleteMark) {
int32_t code = 0;
SNode* pAst = NULL; SNode* pAst = NULL;
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
terrno = TSDB_CODE_SUCCESS;
if (nodesStringToNode(ast, &pAst) < 0) { if (nodesStringToNode(ast, &pAst) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
if (qSetSTableIdForRsma(pAst, uid) < 0) { if (qSetSTableIdForRsma(pAst, uid) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
@ -75,33 +75,33 @@ int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64
}; };
if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) { if (levelNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
int32_t opNum = LIST_LENGTH(inner->pNodeList); int32_t opNum = LIST_LENGTH(inner->pNodeList);
if (opNum != 1) { if (opNum != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
if (qSubPlanToString(plan, pDst, pDstLen) < 0) { if (qSubPlanToString(plan, pDst, pDstLen) < 0) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
goto END; goto END;
} }
END: END:
if (pAst) nodesDestroyNode(pAst); if (pAst) nodesDestroyNode(pAst);
if (pPlan) nodesDestroyNode((SNode*)pPlan); if (pPlan) nodesDestroyNode((SNode*)pPlan);
return terrno; TAOS_RETURN(code);
} }
int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
@ -127,6 +127,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList, int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SArray* pSinkNodeList,
SStreamTask* pTask) { SStreamTask* pTask) {
int32_t code = 0;
bool isShuffle = false; bool isShuffle = false;
if (pStream->fixedSinkVgId == 0) { if (pStream->fixedSinkVgId == 0) {
@ -135,9 +136,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
isShuffle = true; isShuffle = true;
pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->outputInfo.type = TASK_OUTPUT__SHUFFLE_DISPATCH;
pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH;
if (mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL) < 0) { TAOS_CHECK_RETURN(mndExtractDbInfo(pMnode, pDb, &pTask->outputInfo.shuffleDispatcher.dbInfo, NULL));
return -1;
}
} }
sdbRelease(pMnode->pSdb, pDb); sdbRelease(pMnode->pSdb, pDb);
@ -166,7 +165,7 @@ int32_t mndAddDispatcherForInternalTask(SMnode* pMnode, SStreamObj* pStream, SAr
streamTaskSetFixedDownstreamInfo(pTask, pOneSinkTask); streamTaskSetFixedDownstreamInfo(pTask, pOneSinkTask);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) {
@ -639,14 +638,15 @@ static void bindTwoLevel(SArray* tasks, int32_t begin, int32_t end) {
static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey, static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey,
SArray* pVerList) { SArray* pVerList) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb; SSdb* pSdb = pMnode->pSdb;
int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans); int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
bool hasExtraSink = false; bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
if (pDbObj == NULL) { if (pDbObj == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); bool multiTarget = (pDbObj->cfg.numOfVgroups > 1);
@ -670,9 +670,11 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
SSubplan* plan = getScanSubPlan(pPlan); // source plan SSubplan* plan = getScanSubPlan(pPlan); // source plan
if (plan == NULL) { if (plan == NULL) {
return terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int32_t code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1); code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -688,7 +690,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (numOfPlanLevel == 3) { if (numOfPlanLevel == 3) {
plan = getAggSubPlan(pPlan, 1); // middle agg plan plan = getAggSubPlan(pPlan, 1); // middle agg plan
if (plan == NULL) { if (plan == NULL) {
return terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
do { do {
SArray** list = taosArrayGetLast(pStream->tasks); SArray** list = taosArrayGetLast(pStream->tasks);
@ -715,7 +719,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
plan = getAggSubPlan(pPlan, 0); plan = getAggSubPlan(pPlan, 0);
if (plan == NULL) { if (plan == NULL) {
return terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
mDebug("doScheduleStream add final agg"); mDebug("doScheduleStream add final agg");
@ -724,7 +730,7 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
addNewTaskList(pStream); addNewTaskList(pStream);
code = addAggTask(pStream, pMnode, plan, pEpset, true); code = addAggTask(pStream, pMnode, plan, pEpset, true);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; TAOS_RETURN(code);
} }
bindTwoLevel(pStream->tasks, 0, size); bindTwoLevel(pStream->tasks, 0, size);
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
@ -735,26 +741,28 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
if (pStream->conf.fillHistory) { if (pStream->conf.fillHistory) {
bindAggSink(pStream, pMnode, pStream->pHTasksList); bindAggSink(pStream, pMnode, pStream->pHTasksList);
} }
return TDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) { int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) {
int32_t code = 0;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
SEpSet mnodeEpset = {0}; SEpSet mnodeEpset = {0};
mndGetMnodeEpSet(pMnode, &mnodeEpset); mndGetMnodeEpSet(pMnode, &mnodeEpset);
int32_t code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList); code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
return code; TAOS_RETURN(code);
} }
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
int32_t code = 0;
SSdb* pSdb = pMnode->pSdb; SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
@ -763,21 +771,21 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan); pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) { } else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
SNode* pAst = NULL; SNode* pAst = NULL;
if (nodesStringToNode(pTopic->ast, &pAst) != 0) { if ((code = nodesStringToNode(pTopic->ast, &pAst)) != 0) {
mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
return -1; TAOS_RETURN(code);
} }
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) != 0) {
mError("failed to create topic:%s since %s", pTopic->name, terrstr()); mError("failed to create topic:%s since %s", pTopic->name, terrstr());
nodesDestroyNode(pAst); nodesDestroyNode(pAst);
return -1; TAOS_RETURN(code);
} }
nodesDestroyNode(pAst); nodesDestroyNode(pAst);
} }
@ -786,8 +794,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) { if (levelNum != 1) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1; TAOS_RETURN(code);
} }
SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
@ -795,8 +803,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList); int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) { if (opNum != 1) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY; code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1; TAOS_RETURN(code);
} }
pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0); pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
@ -831,13 +839,13 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) { if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT; code = TSDB_CODE_QRY_INVALID_INPUT;
return -1; TAOS_RETURN(code);
} }
} else { } else {
pSub->qmsg = taosStrdup(""); pSub->qmsg = taosStrdup("");
} }
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
return 0; TAOS_RETURN(code);
} }

59
source/dnode/mnode/impl/src/mndShow.c
View File

@ -31,17 +31,18 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow);
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq); static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) { int32_t mndInitShow(SMnode *pMnode) {
int32_t code = 0;
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show"); pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
if (pMgmt->cache == NULL) { if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr()); mError("failed to alloc show cache since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq); mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq);
return 0; TAOS_RETURN(code);
} }
void mndCleanupShow(SMnode *pMnode) { void mndCleanupShow(SMnode *pMnode) {
@ -212,6 +213,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
} }
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowObj *pShow = NULL; SShowObj *pShow = NULL;
@ -220,10 +222,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t rowsRead = 0; int32_t rowsRead = 0;
mDebug("mndProcessRetrieveSysTableReq start"); mDebug("mndProcessRetrieveSysTableReq start");
SRetrieveTableReq retrieveReq = {0}; SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb); mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
@ -232,17 +231,17 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (pMeta == NULL) { if (pMeta == NULL) {
pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) { if (pMeta == NULL) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
} }
pShow = mndCreateShowObj(pMnode, &retrieveReq); pShow = mndCreateShowObj(pMnode, &retrieveReq);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", terrstr()); mError("failed to process show-meta req since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
pShow->pMeta = pMeta; pShow->pMeta = pMeta;
@ -250,9 +249,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; code = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
} }
@ -264,9 +263,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
if (retrieveFp == NULL) { if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
@ -275,14 +274,16 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
} }
if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { code = -1;
return -1; if (retrieveReq.db[0] &&
(code = mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db)) != 0) {
TAOS_RETURN(code);
} }
if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) { if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) {
if(strcmp(pReq->info.conn.user, "root") != 0){ if(strcmp(pReq->info.conn.user, "root") != 0){
mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type); mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type);
terrno = TSDB_CODE_MND_NO_RIGHTS; code = TSDB_CODE_MND_NO_RIGHTS;
return -1; TAOS_RETURN(code);
} }
} }
@ -308,11 +309,11 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead); rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead);
if (rowsRead < 0) { if (rowsRead < 0) {
terrno = rowsRead; code = rowsRead;
mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true); mndReleaseShowObj(pShow, true);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; TAOS_RETURN(code);
} }
pBlock->info.rows = rowsRead; pBlock->info.rows = rowsRead;
@ -325,10 +326,10 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; TAOS_RETURN(code);
} }
pRsp->handle = htobe64(pShow->id); pRsp->handle = htobe64(pShow->id);

703
source/dnode/mnode/impl/src/mndSma.c
View File

File diff suppressed because it is too large Load Diff

186
source/dnode/mnode/impl/src/mndSnode.c
View File

@ -170,38 +170,54 @@ static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew
} }
static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0}; SDCreateSnodeReq createReq = {0};
createReq.dnodeId = pDnode->id; createReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq);
@ -212,23 +228,24 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_CREATE_SNODE; action.msgType = TDMT_DND_CREATE_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +256,12 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_SNODE; action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) { if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) {
@ -256,23 +273,27 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
snodeObj.updateTime = snodeObj.createdTime; snodeObj.updateTime = snodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeRedoLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeUndoLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeCommitLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
@ -282,32 +303,27 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateSnodeReq createReq = {0}; SMCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("snode:%d, start to create", createReq.dnodeId); mInfo("snode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE), NULL, _OVER);
goto _OVER;
}
// pObj = mndAcquireSnode(pMnode, createReq.dnodeId); // pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
// if (pObj != NULL) { // if (pObj != NULL) {
// terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; // terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
// goto _OVER; // goto _OVER;
// } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { // } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) {
// goto _OVER; // goto _OVER;
// } // }
if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){ if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){
terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; code = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
@ -316,41 +332,52 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); mError("snode:%d, failed to create since %s", createReq.dnodeId, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
// mndReleaseSnode(pMnode, pObj); // mndReleaseSnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -361,20 +388,20 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
action.msgType = TDMT_DND_DROP_SNODE; action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) { int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeRedoLogs(pTrans, pObj));
if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeCommitLogs(pTrans, pObj));
if (!force) { if (!force) {
if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj));
} }
return 0; return 0;
} }
@ -383,18 +410,22 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
@ -403,23 +434,20 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
SSnodeObj *pObj = NULL; SSnodeObj *pObj = NULL;
SMDropSnodeReq dropReq = {0}; SMDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("snode:%d, start to drop", dropReq.dnodeId); mInfo("snode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireSnode(pMnode, dropReq.dnodeId); pObj = mndAcquireSnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -429,12 +457,12 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); mError("snode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
} }
mndReleaseSnode(pMnode, pObj); mndReleaseSnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

942
source/dnode/mnode/impl/src/mndStb.c
View File

File diff suppressed because it is too large Load Diff

364
source/dnode/mnode/impl/src/mndStream.c
View File

@ -134,17 +134,18 @@ int32_t mndInitStream(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
mndInitExecInfo(); int32_t code = mndInitExecInfo();
if (code) {
if (sdbSetTable(pMnode->pSdb, table) != 0) { return code;
return -1;
} }
if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { code = sdbSetTable(pMnode->pSdb, table);
return -1; if (code) {
return terrno;
} }
return 0; code = sdbSetTable(pMnode->pSdb, tableSeq);
return code;
} }
void mndCleanupStream(SMnode *pMnode) { void mndCleanupStream(SMnode *pMnode) {
@ -251,13 +252,15 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
return 0; return 0;
} }
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream) {
SSdb *pSdb = pMnode->pSdb; terrno = 0;
SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { SSdb *pSdb = pMnode->pSdb;
(*pStream) = sdbAcquire(pSdb, SDB_STREAM, streamName);
if ((*pStream) == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
} }
return pStream; return terrno;
} }
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
@ -530,9 +533,21 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
} }
int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create task iter for stream:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
}
if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { if (mndPersistTaskDeployReq(pTrans, pTask) < 0) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return -1;
@ -706,7 +721,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
char *sql = NULL; char *sql = NULL;
int32_t sqlLen = 0; int32_t sqlLen = 0;
const char *pMsg = "create stream tasks on dnodes"; const char *pMsg = "create stream tasks on dnodes";
int32_t code = 0;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
SCMCreateStreamReq createReq = {0}; SCMCreateStreamReq createReq = {0};
@ -726,8 +741,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
pStream = mndAcquireStream(pMnode, createReq.name); code = mndAcquireStream(pMnode, createReq.name, &pStream);
if (pStream != NULL) { if (pStream != NULL || code == 0) {
if (createReq.igExists) { if (createReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createReq.name); mInfo("stream:%s, already exist, ignore exist is set", createReq.name);
goto _OVER; goto _OVER;
@ -760,8 +775,9 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg); STrans *pTrans = NULL;
if (pTrans == NULL) { code = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg, &pTrans);
if (pTrans == NULL || code) {
goto _OVER; goto _OVER;
} }
@ -802,11 +818,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
// add into buffer firstly // add into buffer firstly
// to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name); mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name);
saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo);
// mndRegisterConsensusChkptId(execInfo.pStreamConsensus, streamObj.uid); streamMutexUnlock(&execInfo.lock);
taosThreadMutexUnlock(&execInfo.lock);
// execute creation // execute creation
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
@ -867,7 +882,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
{ // check the max checkpoint id from all vnodes. { // check the max checkpoint id from all vnodes.
int64_t maxCheckpointId = -1; int64_t maxCheckpointId = -1;
if (lock) { if (lock) {
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
} }
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
@ -888,7 +903,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
} }
if (lock) { if (lock) {
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
} }
if (maxCheckpointId > maxChkptId) { if (maxCheckpointId > maxChkptId) {
@ -989,11 +1004,13 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, STrans *pTrans = NULL;
"gen checkpoint for stream"); code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME,
if (pTrans == NULL) { "gen checkpoint for stream", &pTrans);
if (pTrans == NULL || code) {
code = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); tstrerror(code));
goto _ERR; goto _ERR;
} }
@ -1033,7 +1050,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) { if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) {
return code; goto _ERR;
} }
if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) { if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) {
@ -1057,13 +1074,13 @@ int32_t extractStreamNodeList(SMnode *pMnode) {
static bool taskNodeIsUpdated(SMnode *pMnode) { static bool taskNodeIsUpdated(SMnode *pMnode) {
// check if the node update happens or not // check if the node update happens or not
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int32_t numOfNodes = extractStreamNodeList(pMnode); int32_t numOfNodes = extractStreamNodeList(pMnode);
if (numOfNodes == 0) { if (numOfNodes == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
execInfo.ts = taosGetTimestampSec(); execInfo.ts = taosGetTimestampSec();
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return false; return false;
} }
@ -1071,17 +1088,22 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i);
if (pNodeEntry->stageUpdated) { if (pNodeEntry->stageUpdated) {
mDebug("stream task not ready due to node update detected, checkpoint not issued"); mDebug("stream task not ready due to node update detected, checkpoint not issued");
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return true; return true;
} }
} }
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
mWarn("not all vnodes ready, quit from vnodes status check"); mWarn("not all vnodes ready, quit from vnodes status check");
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return true; return true;
} }
@ -1097,7 +1119,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
mDebug("stream tasks not ready due to node update"); mDebug("stream tasks not ready due to node update");
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return nodeUpdated; return nodeUpdated;
} }
@ -1107,7 +1129,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
return -1; return -1;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
if (taosArrayGetSize(execInfo.pNodeList) == 0) { if (taosArrayGetSize(execInfo.pNodeList) == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
@ -1152,7 +1174,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
removeTasksInBuf(pInvalidList, &execInfo); removeTasksInBuf(pInvalidList, &execInfo);
taosArrayDestroy(pInvalidList); taosArrayDestroy(pInvalidList);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return ready ? 0 : -1; return ready ? 0 : -1;
} }
@ -1215,14 +1237,14 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
continue; continue;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid); int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid);
if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) { if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) {
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
continue; continue;
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration}; SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration};
taosArrayPush(pList, &in); taosArrayPush(pList, &in);
@ -1265,8 +1287,9 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
for (int32_t i = 0; i < numOfQual; ++i) { for (int32_t i = 0; i < numOfQual; ++i) {
SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i); SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i);
SStreamObj *p = mndGetStreamObj(pMnode, pCheckpointInfo->streamId); SStreamObj *p = NULL;
if (p != NULL) { code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p);
if (p != NULL && code == 0) {
code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true);
sdbRelease(pSdb, p); sdbRelease(pSdb, p);
@ -1289,6 +1312,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
SMDropStreamReq dropReq = {0}; SMDropStreamReq dropReq = {0};
if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) {
@ -1299,8 +1323,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
mDebug("recv drop stream:%s msg", dropReq.name); mDebug("recv drop stream:%s msg", dropReq.name);
pStream = mndAcquireStream(pMnode, dropReq.name); code = mndAcquireStream(pMnode, dropReq.name, &pStream);
if (pStream == NULL) { if (pStream == NULL || code != 0) {
if (dropReq.igNotExists) { if (dropReq.igNotExists) {
mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name); mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1356,15 +1380,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = NULL;
if (pTrans == NULL) { code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code) {
mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr()); mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq); tFreeMDropStreamReq(&dropReq);
return -1; return -1;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid);
// drop all tasks // drop all tasks
if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
@ -1857,9 +1882,9 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfRows = 0; int32_t numOfRows = 0;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo); mndInitStreamExecInfo(pMnode, &execInfo);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
while (numOfRows < rowsCapacity) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
@ -1876,11 +1901,24 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
} }
// add row for each task // add row for each task
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
while (streamTaskIterNextTask(pIter)) { int32_t code = createStreamTaskIter(pStream, &pIter);
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); if (code) {
taosRUnLockLatch(&pStream->lock);
sdbRelease(pSdb, pStream);
mError("failed to create task iter for stream:%s", pStream->name);
continue;
}
int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
break;
}
code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
numOfRows++; numOfRows++;
} }
@ -1906,6 +1944,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter) {
static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
SMPauseStreamReq pauseReq = {0}; SMPauseStreamReq pauseReq = {0};
if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) { if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) {
@ -1913,9 +1952,8 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
pStream = mndAcquireStream(pMnode, pauseReq.name); code = mndAcquireStream(pMnode, pauseReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (pStream == NULL) {
if (pauseReq.igNotExists) { if (pauseReq.igNotExists) {
mInfo("stream:%s, not exist, not pause stream", pauseReq.name); mInfo("stream:%s, not exist, not pause stream", pauseReq.name);
return 0; return 0;
@ -1955,7 +1993,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
{ // check for tasks, if tasks are not ready, not allowed to pause { // check for tasks, if tasks are not ready, not allowed to pause
bool found = false; bool found = false;
bool readyToPause = true; bool readyToPause = true;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
@ -1978,7 +2016,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
found = true; found = true;
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
if (!found) { if (!found) {
mError("stream:%s task not report status yet, not ready for pause", pauseReq.name); mError("stream:%s task not report status yet, not ready for pause", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1992,42 +2030,49 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
} }
} }
STrans *pTrans = STrans *pTrans = NULL;
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream"); code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream", &pTrans);
if (pTrans == NULL) { if (pTrans == NULL || code) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
// if nodeUpdate happened, not send pause trans // if nodeUpdate happened, not send pause trans
if (mndStreamSetPauseAction(pMnode, pTrans, pStream) < 0) { code = mndStreamSetPauseAction(pMnode, pTrans, pStream);
if (code) {
mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr()); mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// pause stream // pause stream
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
pStream->status = STREAM_STATUS__PAUSE; pStream->status = STREAM_STATUS__PAUSE;
if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY) < 0) { code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code) {
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2039,6 +2084,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return -1; return -1;
@ -2050,9 +2096,8 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
pStream = mndAcquireStream(pMnode, resumeReq.name); code = mndAcquireStream(pMnode, resumeReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (pStream == NULL) {
if (resumeReq.igNotExists) { if (resumeReq.igNotExists) {
mInfo("stream:%s not exist, not resume stream", resumeReq.name); mInfo("stream:%s not exist, not resume stream", resumeReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2081,22 +2126,28 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = STrans *pTrans = NULL;
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream"); code =
if (pTrans == NULL) { doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream", &pTrans);
if (pTrans == NULL || code) {
mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
// set the resume action // set the resume action
if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) { if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) {
mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// resume stream // resume stream
@ -2107,7 +2158,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
@ -2115,7 +2166,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2189,6 +2240,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
STrans *pTrans = NULL; STrans *pTrans = NULL;
int32_t code = 0;
// conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool // conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool
while (1) { while (1) {
@ -2215,12 +2267,11 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
// here create only one trans // here create only one trans
if (pTrans == NULL) { if (pTrans == NULL) {
pTrans = code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets", &pTrans);
doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets"); if (pTrans == NULL || code) {
if (pTrans == NULL) {
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return terrno; return terrno = code;
} }
mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid);
@ -2237,7 +2288,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
pStream->name, pTrans->id); pStream->name, pTrans->id);
int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans);
// todo: not continue, drop all and retry again // todo: not continue, drop all and retry again
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -2252,7 +2303,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; return code;
} }
} }
@ -2261,16 +2312,17 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0; return 0;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
@ -2287,9 +2339,21 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
SStreamTaskIter *pTaskIter = createStreamTaskIter(pStream); SStreamTaskIter *pTaskIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pTaskIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
sdbRelease(pSdb, pStream);
mError("failed to create task iter for stream:%s", pStream->name);
continue;
}
while (streamTaskIterNextTask(pTaskIter)) { while (streamTaskIterNextTask(pTaskIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pTaskIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pTaskIter, &pTask);
if (code) {
break;
}
SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
epsetAssign(&entry.epset, &pTask->info.epSet); epsetAssign(&entry.epset, &pTask->info.epSet);
@ -2336,9 +2400,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int32_t numOfNodes = extractStreamNodeList(pMnode); int32_t numOfNodes = extractStreamNodeList(pMnode);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
if (numOfNodes == 0) { if (numOfNodes == 0) {
mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing"); mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing");
@ -2348,7 +2412,13 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
} }
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
if (code) {
mError("failed to take the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
atomic_store_32(&mndNodeCheckSentinel, 0); atomic_store_32(&mndNodeCheckSentinel, 0);
@ -2356,7 +2426,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
return 0; return 0;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot);
@ -2380,7 +2450,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
} }
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(changeInfo.pUpdateNodeList); taosArrayDestroy(changeInfo.pUpdateNodeList);
taosHashCleanup(changeInfo.pDBMap); taosHashCleanup(changeInfo.pDBMap);
@ -2406,9 +2476,19 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
} }
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create task iter for stream:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
break;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
@ -2478,10 +2558,11 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId); mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream);
if (pStream == NULL || code != 0) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf", mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf",
req.streamId); req.streamId);
@ -2492,7 +2573,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return -1; return -1;
} else { } else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
@ -2537,7 +2618,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
{ {
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)}; SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)};
@ -2597,10 +2678,11 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId); req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream);
if (pStream == NULL || code != 0) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId); mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer // not in meta-store yet, try to acquire the task in exec buffer
@ -2610,7 +2692,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return -1; return -1;
} else { } else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
@ -2642,7 +2724,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS); doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS);
return 0; return 0;
@ -2707,7 +2789,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// req.nodeId, req.streamId, req.taskId, req.checkpointId); // req.nodeId, req.streamId, req.taskId, req.checkpointId);
// //
// // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
// taosThreadMutexLock(&execInfo.lock); // streamMutexLock(&execInfo.lock);
// //
// // mnode handle the create stream transaction too slow may cause this problem // // mnode handle the create stream transaction too slow may cause this problem
// SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); // SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
@ -2721,7 +2803,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// if (p == NULL) { // if (p == NULL) {
// mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId); // mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId);
// terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; // terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// //
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return -1; // return -1;
@ -2737,7 +2819,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// //
// int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream); // int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
// if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly // if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs);
// //
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
@ -2754,7 +2836,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks); // SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks);
// mndAddConsensusTasks(pInfo, &req); // mndAddConsensusTasks(pInfo, &req);
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
// } // }
@ -2764,7 +2846,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId); // req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId);
// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs);
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
// } // }
@ -2777,7 +2859,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// mndReleaseStream(pMnode, pStream); // mndReleaseStream(pMnode, pStream);
// } // }
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
//} //}
@ -2790,15 +2872,21 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
mDebug("start to process consensus-checkpointId in tmr"); mDebug("start to process consensus-checkpointId in tmr");
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process"); mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process");
taosArrayDestroy(pStreamList); taosArrayDestroy(pStreamList);
return 0; return 0;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
void *pIter = NULL; void *pIter = NULL;
while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) {
@ -2808,8 +2896,9 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
int32_t num = taosArrayGetSize(pInfo->pTaskList); int32_t num = taosArrayGetSize(pInfo->pTaskList);
SArray *pList = taosArrayInit(4, sizeof(int32_t)); SArray *pList = taosArrayInit(4, sizeof(int32_t));
SStreamObj *pStream = mndGetStreamObj(pMnode, pInfo->streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { // stream has been dropped already code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) { // stream has been dropped already
mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId); mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId);
taosArrayDestroy(pList); taosArrayDestroy(pList);
continue; continue;
@ -2868,14 +2957,14 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) {
int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i); int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i);
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId); code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(pStreamList); taosArrayDestroy(pStreamList);
mDebug("end to process consensus-checkpointId in tmr"); mDebug("end to process consensus-checkpointId in tmr");
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) { static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
@ -2926,32 +3015,41 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
} }
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) { int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, STrans *pTrans = NULL;
"update checkpoint-info"); int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME,
if (pTrans == NULL) { "update checkpoint-info", &pTrans);
return terrno; if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
} }
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); if (code){
if (code != 0) { sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
} }
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) { if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);

104
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -61,15 +61,23 @@ void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
} }
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, STrans *pTrans = NULL;
" reset from failed checkpoint"); int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
if (pTrans == NULL) { " reset from failed checkpoint", &pTrans);
if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return terrno; return terrno;
} }
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); if (code) {
if (code != 0) { sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
@ -79,14 +87,15 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -99,8 +108,9 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t t
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
mndKillTransImpl(pMnode, transId, ""); mndKillTransImpl(pMnode, transId, "");
SStreamObj *pStream = mndGetStreamObj(pMnode, streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { code = mndGetStreamObj(pMnode, streamId, &pStream);
if (pStream == NULL || code != 0) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST; code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid);
} else { } else {
@ -159,34 +169,39 @@ int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) {
} }
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = NULL;
if (pTrans == NULL) { int32_t code =
doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code != 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
if (code) {
return code;
}
// drop all tasks // drop all tasks
if (mndStreamSetDropActionFromList(pMnode, pTrans, pList) < 0) { if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, pList)) < 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// drop stream // drop stream
if (mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED) < 0) { if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) {
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
@ -228,10 +243,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SStreamHbMsg req = {0}; SStreamHbMsg req = {0};
SArray *pFailedChkpt = NULL; SArray *pFailedChkpt = NULL;
SArray *pOrphanTasks = NULL; SArray *pOrphanTasks = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if (suspendAllStreams(pMnode, &pReq->info) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) {
return -1; return code;
} }
} }
@ -241,8 +257,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (tDecodeStreamHbMsg(&decoder, &req) < 0) { if (tDecodeStreamHbMsg(&decoder, &req) < 0) {
tCleanupStreamHbMsg(&req); tCleanupStreamHbMsg(&req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG; code = terrno = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -257,12 +273,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { if (!validateHbMsg(execInfo.pNodeList, req.vgId)) {
mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); mError("vgId:%d not exists in nodeList buf, discarded", req.vgId);
terrno = TSDB_CODE_INVALID_MSG; code = terrno = TSDB_CODE_INVALID_MSG;
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return -1; return code;
} }
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
@ -293,11 +309,23 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
.startTs = pChkInfo->consensusTs, .startTs = pChkInfo->consensusTs,
}; };
SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId); SStreamObj *pStream = NULL;
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); code = mndGetStreamObj(pMnode, p->id.streamId, &pStream);
if (code) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
continue;
}
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
SCheckpointConsensusInfo *pInfo = NULL;
code = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks, &pInfo);
if (code == 0) {
mndAddConsensusTasks(pInfo, &cp);
} else {
mError("failed to get consensus checkpoint-info");
}
SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks);
mndAddConsensusTasks(pInfo, &cp);
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
@ -338,9 +366,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
// kill the checkpoint trans and then set all tasks status to be normal // kill the checkpoint trans and then set all tasks status to be normal
if (taosArrayGetSize(pFailedChkpt) > 0) { if (taosArrayGetSize(pFailedChkpt) > 0) {
bool allReady = true; bool allReady = true;
if (pMnode != NULL) { if (pMnode != NULL) {
SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *p = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &p);
taosArrayDestroy(p); taosArrayDestroy(p);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
} else { } else {
allReady = false; allReady = false;
} }
@ -374,7 +408,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return TSDB_CODE_SUCCESS; return terrno;
} }
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg

32
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -153,27 +153,30 @@ int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) {
return 0; return 0;
} }
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg) { const char *pMsg, STrans ** pTrans1) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); *pTrans1 = NULL;
if (pTrans == NULL) { terrno = 0;
STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (p == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return terrno;
} }
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName); mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, p) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT; terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno)); mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
mndTransDrop(pTrans); mndTransDrop(p);
return NULL; return terrno;
} }
terrno = 0; *pTrans1 = p;
return pTrans; return 0;
} }
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@ -272,8 +275,9 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) {
continue; continue;
} }
SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId); SStreamObj *pStream = NULL;
if (pStream != NULL) { int32_t code = mndGetStreamObj(pMnode, pTransInfo->streamId, &pStream);
if (pStream != NULL && code == 0) {
if (identicalName(pStream->sourceDb, pDBName, len)) { if (identicalName(pStream->sourceDb, pDBName, len)) {
mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb);
} else if (identicalName(pStream->targetDb, pDBName, len)) { } else if (identicalName(pStream->targetDb, pDBName, len)) {

541
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -28,20 +28,20 @@ struct SStreamTaskIter {
int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId); int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId);
SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) { int32_t createStreamTaskIter(SStreamObj* pStream, SStreamTaskIter** pIter) {
SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter)); *pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
if (pIter == NULL) { if (*pIter == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return terrno;
} }
pIter->level = -1; (*pIter)->level = -1;
pIter->ordinalIndex = 0; (*pIter)->ordinalIndex = 0;
pIter->pStream = pStream; (*pIter)->pStream = pStream;
pIter->totalLevel = taosArrayGetSize(pStream->tasks); (*pIter)->totalLevel = taosArrayGetSize(pStream->tasks);
pIter->pTask = NULL; (*pIter)->pTask = NULL;
return pIter; return 0;
} }
bool streamTaskIterNextTask(SStreamTaskIter* pIter) { bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
@ -72,19 +72,27 @@ bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
return false; return false;
} }
SStreamTask* streamTaskIterGetCurrent(SStreamTaskIter* pIter) { int32_t streamTaskIterGetCurrent(SStreamTaskIter* pIter, SStreamTask** pTask) {
return pIter->pTask; if (pTask) {
*pTask = pIter->pTask;
if (*pTask != NULL) {
return TSDB_CODE_SUCCESS;
}
}
return TSDB_CODE_INVALID_PARA;
} }
void destroyStreamTaskIter(SStreamTaskIter* pIter) { void destroyStreamTaskIter(SStreamTaskIter* pIter) {
taosMemoryFree(pIter); taosMemoryFree(pIter);
} }
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
int32_t replica = -1; // do the replica check int32_t replica = -1; // do the replica check
int32_t code = 0;
*allReady = true; *allReady = true;
SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry)); SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry));
@ -131,10 +139,15 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
} }
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf)); (void) epsetToStr(&entry.epset, buf, tListLen(buf));
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
}
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -145,43 +158,57 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
break; break;
} }
SNodeEntry entry = {0}; SNodeEntry entry = {.nodeId = SNODE_HANDLE};
addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port);
entry.nodeId = SNODE_HANDLE; if (code) {
sdbRelease(pSdb, pObj);
continue;
}
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf)); (void) epsetToStr(&entry.epset, buf, tListLen(buf));
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
}
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pObj); sdbRelease(pSdb, pObj);
} }
return pVgroupList; *pList = pVgroupList;
return code;
} }
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj **pStream) {
void *pIter = NULL; void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; *pStream = NULL;
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { SStreamObj *p = NULL;
if (pStream->uid == streamId) { while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&p)) != NULL) {
if (p->uid == streamId) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return pStream; *pStream = p;
return TSDB_CODE_SUCCESS;
} }
sdbRelease(pSdb, pStream); sdbRelease(pSdb, p);
} }
return NULL; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) { void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) {
STrans *pTrans = mndAcquireTrans(pMnode, transId); STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans != NULL) { if (pTrans != NULL) {
mInfo("kill active transId:%d in Db:%s", transId, pDbName); mInfo("kill active transId:%d in Db:%s", transId, pDbName);
mndKillTrans(pMnode, pTrans); int32_t code = mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans);
if (code) {
mError("failed to kill trans:%d", pTrans->id);
}
} else { } else {
mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId); mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId);
} }
@ -197,11 +224,16 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj);
if (pIter != NULL) { if (pIter != NULL) {
addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); int32_t code = addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
sdbRelease(pMnode->pSdb, pObj); sdbRelease(pMnode->pSdb, pObj);
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
*hasEpset = true; if (code) {
return TSDB_CODE_SUCCESS; *hasEpset = false;
mError("failed to set epset");
} else {
*hasEpset = true;
}
return code;
} else { } else {
mError("failed to acquire snode epset"); mError("failed to acquire snode epset");
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -223,12 +255,14 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
} }
static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) {
terrno = 0;
SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -242,31 +276,45 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { if (code != TSDB_CODE_SUCCESS || (!hasEpset)) {
terrno = code; terrno = code;
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return terrno;
} }
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return terrno;
} }
mDebug("set the resume action for trans:%d", pTrans->id); mDebug("set the resume action for trans:%d", pTrans->id);
return 0; return 0;
} }
SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); *pTask = NULL;
SStreamTask *p = NULL;
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); code = streamTaskIterGetCurrent(pIter, &p);
if (pTask->id.taskId == pId->taskId) { if (code) {
continue;
}
if (p->id.taskId == pId->taskId) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return pTask; *pTask = p;
return 0;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return NULL; return TSDB_CODE_FAILED;
} }
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) { int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
@ -280,13 +328,25 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
} }
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) { int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code || pTask == NULL) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetResumeAction(pTrans, pMnode, pTask, igUntreated);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) {
@ -303,7 +363,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq), mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -320,25 +380,38 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
} }
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&epset, buf, tListLen(buf)); (void) epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
} }
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetPauseAction(pMnode, pTrans, pTask) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetPauseAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) {
@ -348,14 +421,14 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return 0; return code;
} }
static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -366,28 +439,40 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction
terrno = code; return code;
return -1;
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
} }
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while(streamTaskIterNextTask(pIter)) { while(streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetDropAction(pMnode, pTrans, pTask) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetDropAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
@ -398,7 +483,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->nodeId); pReq->head.vgId = htonl(pTask->nodeId);
@ -409,16 +494,15 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId); int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId);
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction
terrno = code;
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
@ -427,19 +511,35 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) { int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) {
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SOrphanTask* pTask = taosArrayGet(pList, i); SOrphanTask* pTask = taosArrayGet(pList, i);
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId); int32_t code = doSetDropActionFromId(pMnode, pTrans, pTask);
doSetDropActionFromId(pMnode, pTrans, pTask); if (code != 0) {
return code;
} else {
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId);
}
} }
return 0; return 0;
} }
static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId, static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
int32_t transId) { int32_t transId) {
int32_t code = 0;
pMsg->streamId = pId->streamId; pMsg->streamId = pId->streamId;
pMsg->taskId = pId->taskId; pMsg->taskId = pId->taskId;
pMsg->transId = transId; pMsg->transId = transId;
pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo)); pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList); if (pMsg->pNodeList == NULL) {
mError("failed to prepare node list, code:out of memory");
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code == 0) {
void *p = taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
if (p == NULL) {
mError("failed to add update node list into nodeList");
}
}
} }
static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
@ -454,7 +554,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return -1; return terrno;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
@ -463,13 +563,18 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return -1; return terrno;
} }
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
tEncodeStreamTaskUpdateMsg(&encoder, &req); code = tEncodeStreamTaskUpdateMsg(&encoder, &req);
if (code == -1) {
tEncoderClear(&encoder);
taosArrayDestroy(req.pNodeList);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)buf; SMsgHead *pMsgHead = (SMsgHead *)buf;
pMsgHead->contLen = htonl(tlen); pMsgHead->contLen = htonl(tlen);
@ -487,15 +592,20 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
void *pBuf = NULL; void *pBuf = NULL;
int32_t len = 0; int32_t len = 0;
streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); int32_t code = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
if (code) {
return code;
}
doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
if (code) {
return code;
}
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { if (code != TSDB_CODE_SUCCESS || !hasEpset) {
terrno = code;
return code; return code;
} }
@ -510,16 +620,30 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
// build trans to update the epset // build trans to update the epset
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
@ -558,16 +682,30 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
} }
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
int32_t code = doSetResetAction(pMnode, pTrans, pTask); code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetResetAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
@ -581,8 +719,12 @@ static void freeTaskList(void* param) {
taosArrayDestroy(*pList); taosArrayDestroy(*pList);
} }
void mndInitExecInfo() { int32_t mndInitExecInfo() {
taosThreadMutexInit(&execInfo.lock, NULL); int32_t code = taosThreadMutexInit(&execInfo.lock, NULL);
if (code) {
return code;
}
_hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
@ -596,6 +738,7 @@ void mndInitExecInfo() {
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList); taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList); taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList);
return 0;
} }
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
@ -608,7 +751,10 @@ void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
for (int32_t j = 0; j < size; ++j) { for (int32_t j = 0; j < size; ++j) {
SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
if (pEntry->nodeId == p->nodeId) { if (pEntry->nodeId == p->nodeId) {
taosArrayPush(pValidList, p); void* px = taosArrayPush(pValidList, p);
if (px == NULL) {
mError("failed to put node into list, nodeId:%d", p->nodeId);
}
break; break;
} }
} }
@ -626,7 +772,10 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); int32_t code = taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
if (code) {
return code;
}
for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
@ -645,28 +794,45 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) { void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) {
STaskId *pId = taosArrayGet(pTaskIds, i); STaskId *pId = taosArrayGet(pTaskIds, i);
doRemoveTasks(pExecInfo, pId); int32_t code = doRemoveTasks(pExecInfo, pId);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, pId->taskId);
}
} }
} }
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexLock(&pExecNode->lock); SStreamTaskIter *pIter = NULL;
streamMutexLock(&pExecNode->lock);
// 1. remove task entries // 1. remove task entries
SStreamTaskIter *pIter = createStreamTaskIter(pStream); int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
streamMutexUnlock(&pExecNode->lock);
mError("failed to create stream task iter:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
continue;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
doRemoveTasks(pExecNode, &id); code = doRemoveTasks(pExecNode, &id);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, id.taskId);
}
} }
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
// 2. remove stream entry in consensus hash table // 2. remove stream entry in consensus hash table
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); (void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid);
taosThreadMutexUnlock(&pExecNode->lock); streamMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
} }
@ -697,7 +863,10 @@ int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) { if (!existed) {
taosArrayPush(pRemovedTasks, pId); void* p = taosArrayPush(pRemovedTasks, pId);
if (p == NULL) {
mError("failed to put task entry into remove list, taskId:0x%" PRIx64, pId->taskId);
}
} }
} }
@ -759,45 +928,64 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas
} }
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask); code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return 0; return code;
} }
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
void *pIter = NULL; void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t)); SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
int32_t code = 0;
mDebug("start to scan checkpoint report info"); mDebug("start to scan checkpoint report info");
while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) {
SArray *pList = *(SArray **)pIter; SArray *pList = *(SArray **)pIter;
STaskChkptInfo* pInfo = taosArrayGet(pList, 0); STaskChkptInfo *pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId); mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId); void* p = taosArrayPush(pDropped, &pInfo->streamId);
if (p == NULL) {
mError("failed to put stream into drop list:0x%" PRIx64, pInfo->streamId);
}
continue; continue;
} }
int32_t total = mndGetNumOfStreamTasks(pStream); int32_t total = mndGetNumOfStreamTasks(pStream);
int32_t existed = (int32_t) taosArrayGetSize(pList); int32_t existed = (int32_t)taosArrayGetSize(pList);
if (total == existed) { if (total == existed) {
mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info", mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info",
@ -805,17 +993,21 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) { if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId); void* p = taosArrayPush(pDropped, &pInfo->streamId);
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId); if (p == NULL) {
mError("failed to remove stream:0x%" PRIx64, pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
}
} else { } else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet", mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId); pInfo->streamId);
} }
break; break;
} else { } else {
mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); mDebug("stream:0x%" PRIx64 " active checkpoint trans not finished yet, wait", pInfo->streamId);
} }
} else { } else {
mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name, mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name,
@ -829,7 +1021,10 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
if (size > 0) { if (size > 0) {
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i); int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); code = taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
if (code) {
mError("failed to remove stream in buf:0x%"PRIx64, streamId);
}
} }
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
@ -854,29 +1049,30 @@ static int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStream
int32_t blen; int32_t blen;
tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code); tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code);
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
void *pBuf = taosMemoryMalloc(tlen); void *pBuf = taosMemoryMalloc(tlen);
if (pBuf == NULL) { if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
tEncodeRestoreCheckpointInfo(&encoder, &req); code = tEncodeRestoreCheckpointInfo(&encoder, &req);
tEncoderClear(&encoder);
if (code == -1) {
taosMemoryFree(pBuf);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)pBuf; SMsgHead *pMsgHead = (SMsgHead *)pBuf;
pMsgHead->contLen = htonl(tlen); pMsgHead->contLen = htonl(tlen);
pMsgHead->vgId = htonl(pTask->info.nodeId); pMsgHead->vgId = htonl(pTask->info.nodeId);
tEncoderClear(&encoder);
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
@ -898,17 +1094,28 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
char msg[128] = {0}; char msg[128] = {0};
snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId); snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId);
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg); STrans *pTrans = NULL;
if (pTrans == NULL) { int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg, &pTrans);
if (pTrans == NULL || code != 0) {
return terrno; return terrno;
} }
STaskId id = {.streamId = pStream->uid, .taskId = taskId}; STaskId id = {.streamId = pStream->uid, .taskId = taskId};
SStreamTask *pTask = mndGetStreamTask(&id, pStream); SStreamTask *pTask = NULL;
ASSERT(pTask); code = mndGetStreamTask(&id, pStream, &pTask);
if (code) {
mError("failed to get task:0x%x in stream:%s, failed to create consensus-checkpointId", taskId, pStream->name);
sdbRelease(pMnode->pSdb, pStream);
return code;
}
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid);
int32_t code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts); if (code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
}
code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts);
if (code != 0) { if (code != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -916,17 +1123,18 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
} }
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) { if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -935,10 +1143,13 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks) { int32_t mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo) {
void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); *pInfo = NULL;
if (pInfo != NULL) {
return (SCheckpointConsensusInfo*)pInfo; void* px = taosHashGet(pHash, &streamId, sizeof(streamId));
if (px != NULL) {
*pInfo = px;
return 0;
} }
SCheckpointConsensusInfo p = { SCheckpointConsensusInfo p = {
@ -947,10 +1158,14 @@ SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId,
.streamId = streamId, .streamId = streamId,
}; };
taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p)); int32_t code = taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
if (code == 0) {
void* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId)); void *pChkptInfo = (SCheckpointConsensusInfo *)taosHashGet(pHash, &streamId, sizeof(streamId));
return pChkptInfo; *pInfo = pChkptInfo;
} else {
*pInfo = NULL;
}
return code;
} }
// no matter existed or not, add the request into info list anyway, since we need to send rsp mannually // no matter existed or not, add the request into info list anyway, since we need to send rsp mannually
@ -971,11 +1186,15 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo
} }
} }
taosArrayPush(pInfo->pTaskList, &info); void *p = taosArrayPush(pInfo->pTaskList, &info);
int32_t num = taosArrayGetSize(pInfo->pTaskList); if (p == NULL) {
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64 mError("s-task:0x%x failed to put task into consensus-checkpointId list, code: out of memory", info.req.taskId);
" waiting tasks:%d", } else {
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num); int32_t num = taosArrayGetSize(pInfo->pTaskList);
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64
" waiting tasks:%d",
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num);
}
} }
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
@ -984,22 +1203,14 @@ void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
} }
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {
taosHashRemove(pHash, &streamId, sizeof(streamId)); int32_t code = taosHashRemove(pHash, &streamId, sizeof(streamId));
int32_t numOfStreams = taosHashGetSize(pHash); if (code == 0) {
mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", streamId, int32_t numOfStreams = taosHashGetSize(pHash);
numOfStreams); mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d",
return TSDB_CODE_SUCCESS; streamId, numOfStreams);
} } else {
mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list", streamId);
}
//int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) { return code;
// void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); }
// ASSERT(pInfo == NULL);
//
// SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL};
// taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
//
// SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
// ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0);
// mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId);
// return TSDB_CODE_SUCCESS;
//}

12
source/dnode/snode/src/snode.c
View File

@ -61,6 +61,7 @@ int32_t sndBuildStreamTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProce
} }
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
int32_t code = 0;
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) { if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -68,12 +69,17 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
} }
stopRsync(); stopRsync();
startRsync(); code = startRsync();
if (code != 0) {
terrno = code;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
int32_t code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE, code = streamMetaOpen(path, pSnode, (FTaskBuild *)sndBuildStreamTask, tqExpandStreamTask, SNODE_HANDLE,
taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta); taosGetTimestampMs(), tqStartTaskCompleteCallback, &pSnode->pMeta);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto FAIL; goto FAIL;
} }

4
source/dnode/vnode/src/inc/tq.h
View File

@ -155,8 +155,8 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_
SSubmitTbData* pTableData, int64_t earlyTs, const char* id); SSubmitTbData* pTableData, int64_t earlyTs, const char* id);
int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq);
#define TQ_ERR_GO_TO_END(c) \ #define TQ_ERR_GO_TO_END(c) \
do { \ do { \

16
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -23,7 +23,9 @@ static int32_t rsmaRestore(SSma *pSma);
#define SMA_SET_KEEP_CFG(v, l) \ #define SMA_SET_KEEP_CFG(v, l) \
do { \ do { \
SRetention *r = &pCfg->retentions[l]; \ SRetention *r = &pCfg->retentions[l]; \
pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ int64_t keep = -1; \
convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep); \
pKeepCfg->keep2 = (int32_t)keep; \
pKeepCfg->keep0 = pKeepCfg->keep2; \ pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \ pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \ pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \
@ -60,8 +62,12 @@ static int32_t rsmaRestore(SSma *pSma);
* @return int32_t * @return int32_t
*/ */
static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) { static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) {
int32_t freqDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE); int32_t code = TSDB_CODE_SUCCESS;
int32_t keepDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE);
int64_t freqDuration = -1;
int64_t keepDuration = -1;
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t days = duration; // min int32_t days = duration; // min
if (days < freqDuration) { if (days < freqDuration) {
@ -76,8 +82,8 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
goto _exit; goto _exit;
} }
freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); code = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); code = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level
days *= (nFreqTimes > 1 ? nFreqTimes : 1); days *= (nFreqTimes > 1 ? nFreqTimes : 1);

5
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -335,8 +335,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
} }
if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
int64_t msInterval = int64_t msInterval = -1;
convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); TAOS_CHECK_RETURN(convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision,
TIME_UNIT_MILLISECOND, &msInterval));
pItem->maxDelay = (int32_t)msInterval; pItem->maxDelay = (int32_t)msInterval;
} else { } else {
pItem->maxDelay = (int32_t)param->maxdelay[idx]; pItem->maxDelay = (int32_t)param->maxdelay[idx];

18
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -69,8 +69,9 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
} }
STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg;
int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); int64_t sInterval = -1;
if (sInterval <= 0) { code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval);
if (TSDB_CODE_SUCCESS != code || 0 == sInterval) {
*days = pTsdbCfg->days; *days = pTsdbCfg->days;
goto _exit; goto _exit;
} }
@ -78,7 +79,11 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
if (records >= SMA_STORAGE_SPLIT_FACTOR) { if (records >= SMA_STORAGE_SPLIT_FACTOR) {
*days = pTsdbCfg->days; *days = pTsdbCfg->days;
} else { } else {
int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); int64_t mInterval = -1;
code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2;
if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { if (daysPerFile > SMA_STORAGE_MINUTES_MAX) {
@ -196,7 +201,12 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE}; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
code = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq);
if (code) {
smaError("failed to build create-table req, code:%d", code);
continue;
}
{ {
uint64_t groupId = pDataBlock->info.id.groupId; uint64_t groupId = pDataBlock->info.id.groupId;

51
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -103,31 +103,31 @@ int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL; TXN* txn = NULL;
TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_RETURN(tdbTbUpsert(ttb, key, kLen, value, vLen, txn)); TQ_ERR_GO_TO_END(tdbTbUpsert(ttb, key, kLen, value, vLen, txn));
TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0; return 0;
END:
tdbAbort(pTq->pMetaDB, txn);
return code;
} }
int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) { int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) {
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
TXN* txn = NULL; TXN* txn = NULL;
TQ_ERR_RETURN(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); TQ_ERR_GO_TO_END(tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_RETURN(tdbTbDelete(ttb, key, kLen, txn)); TQ_ERR_GO_TO_END(tdbTbDelete(ttb, key, kLen, txn));
TQ_ERR_RETURN(tdbCommit(pTq->pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbCommit(pTq->pMetaDB, txn));
TQ_ERR_RETURN(tdbPostCommit(pTq->pMetaDB, txn)); TQ_ERR_GO_TO_END(tdbPostCommit(pTq->pMetaDB, txn));
return 0; return 0;
}
static int32_t tqMetaTransformOffsetInfo(STQ* pTq, char* path) {
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_RETURN(tqOffsetRestoreFromFile(pTq, path));
END: END:
tdbAbort(pTq->pMetaDB, txn);
return code; return code;
} }
@ -408,7 +408,8 @@ END:
} }
int32_t tqMetaOpen(STQ* pTq) { int32_t tqMetaOpen(STQ* pTq) {
char* maindb = NULL; char* maindb = NULL;
char* offsetNew = NULL;
int32_t code = TDB_CODE_SUCCESS; int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME)); TQ_ERR_GO_TO_END(tqBuildFName(&maindb, pTq->path, TDB_MAINDB_NAME));
if(!taosCheckExistFile(maindb)){ if(!taosCheckExistFile(maindb)){
@ -416,12 +417,20 @@ int32_t tqMetaOpen(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq)); TQ_ERR_GO_TO_END(tqMetaOpenTdb(pTq));
}else{ }else{
TQ_ERR_GO_TO_END(tqMetaTransform(pTq)); TQ_ERR_GO_TO_END(tqMetaTransform(pTq));
taosRemoveFile(maindb); (void)taosRemoveFile(maindb);
} }
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offsetNew)){
TQ_ERR_GO_TO_END(tqOffsetRestoreFromFile(pTq, offsetNew));
(void)taosRemoveFile(offsetNew);
}
TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq)); TQ_ERR_GO_TO_END(tqMetaRestoreCheckInfo(pTq));
END: END:
taosMemoryFree(maindb); taosMemoryFree(maindb);
taosMemoryFree(offsetNew);
return code; return code;
} }
@ -445,14 +454,14 @@ int32_t tqMetaTransform(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore)); TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME)); TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offset) && taosCopyFile(offset, offsetNew) < 0){ if(taosCheckExistFile(offset)) {
tqError("copy offset file error"); if (taosCopyFile(offset, offsetNew) < 0) {
tqError("copy offset file error");
} else {
(void)taosRemoveFile(offset);
}
} }
TQ_ERR_GO_TO_END(tqMetaTransformOffsetInfo(pTq, offsetNew));
(void)taosRemoveFile(offset);
(void)taosRemoveFile(offsetNew);
END: END:
taosMemoryFree(offset); taosMemoryFree(offset);
taosMemoryFree(offsetNew); taosMemoryFree(offsetNew);

1
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -73,6 +73,7 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
} }
TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size)); TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size));
tqInfo("tq: offset restore from file to tdb, subkey:%s", offset.subKey);
taosMemoryFree(pMemBuf); taosMemoryFree(pMemBuf);
pMemBuf = NULL; pMemBuf = NULL;
} }

296
source/dnode/vnode/src/tq/tqSink.c
View File

@ -18,8 +18,6 @@
#include "tmsg.h" #include "tmsg.h"
#include "tq.h" #include "tq.h"
#define MAX_CACHE_TABLE_INFO_NUM 10240
typedef struct STableSinkInfo { typedef struct STableSinkInfo {
uint64_t uid; uint64_t uid;
tstr name; tstr name;
@ -43,7 +41,7 @@ static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, con
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
int32_t numOfTags); int32_t numOfTags);
static SArray* createDefaultTagColName(); static int32_t createDefaultTagColName(SArray** pList);
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule); int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo); static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
@ -95,7 +93,10 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
SSingleDeleteReq req = {.startTs = skey, .endTs = ekey}; SSingleDeleteReq req = {.startTs = skey, .endTs = ekey};
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1);
taosArrayPush(deleteReq->deleteReqs, &req); void* p = taosArrayPush(deleteReq->deleteReqs, &req);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (originName) name = originName; if (originName) name = originName;
taosMemoryFreeClear(name); taosMemoryFreeClear(name);
@ -149,14 +150,20 @@ static bool tqGetTableInfo(SSHashObj* pTableInfoMap, uint64_t groupId, STableSin
static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
void* buf = NULL; void* buf = NULL;
int32_t tlen = 0; int32_t tlen = 0;
encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
if (code) {
tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code));
return code;
}
SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen}; SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen};
if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg);
if (code) {
tqError("failed to put into write-queue since %s", terrstr()); tqError("failed to put into write-queue since %s", terrstr());
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) { int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
@ -166,18 +173,36 @@ int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const
// set super table name // set super table name
SName name = {0}; SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); int32_t code = tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if (code == 0) {
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code
tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName);
}
}
pCreateTableReq->ctb.tagNum = numOfTags; pCreateTableReq->ctb.tagNum = numOfTags;
return TSDB_CODE_SUCCESS; return code;
} }
SArray* createDefaultTagColName() { int32_t createDefaultTagColName(SArray** pColNameList) {
*pColNameList = NULL;
SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN); SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; if (pTagColNameList == NULL) {
taosArrayPush(pTagColNameList, tagNameStr); return TSDB_CODE_OUT_OF_MEMORY;
return pTagColNameList; }
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
void* p = taosArrayPush(pTagColNameList, tagNameStr);
if (p == NULL) {
taosArrayDestroy(pTagColNameList);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pColNameList = pTagColNameList;
return TSDB_CODE_SUCCESS;
} }
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
@ -201,18 +226,20 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
SStreamTask* pTask, int64_t suid) { SStreamTask* pTask, int64_t suid) {
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId;
tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows); tqDebug("s-task:%s build create %d table(s) msg", id, rows);
int32_t code = 0; int32_t code = 0;
SVCreateTbBatchReq reqs = {0}; SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) { if (NULL == reqs.pArray) {
tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno)); tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno));
goto _end; goto _end;
} }
@ -222,15 +249,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1); int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags); code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
if (code) {
tqError("s-task:%s vgId:%d failed to init create table msg", id, vgId);
continue;
}
taosArrayClear(tagArray); taosArrayClear(tagArray);
if (size == 2) { if (size == 2) {
STagVal tagVal = { STagVal tagVal = {
.cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(tagArray, &tagVal); void* p = taosArrayPush(tagArray, &tagVal);
pCreateTbReq->ctb.tagName = createDefaultTagColName(); if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
if (code) {
return code;
}
} else { } else {
for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
@ -245,14 +283,19 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
} else { } else {
memcpy(&tagVal.i64, pData, pTagData->info.bytes); memcpy(&tagVal.i64, pData, pTagData->info.bytes);
} }
taosArrayPush(tagArray, &tagVal); void* p = taosArrayPush(tagArray, &tagVal);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
} }
} }
tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); code = tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
taosArrayDestroy(tagArray); taosArrayDestroy(tagArray);
tagArray = NULL; tagArray = NULL;
if (pCreateTbReq->ctb.pTag == NULL) {
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
@ -270,22 +313,34 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid, setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq); void* p = taosArrayPush(reqs.pArray, pCreateTbReq);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
STableSinkInfo* pInfo = NULL; STableSinkInfo* pInfo = NULL;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo); bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo);
if (!alreadyCached) { if (!alreadyCached) {
code = doCreateSinkInfo(pCreateTbReq->name, &pInfo); code = doCreateSinkInfo(pCreateTbReq->name, &pInfo);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr); if (code) {
tqError("vgId:%d failed to create sink tableInfo for table:%s, s-task:%s", vgId, pCreateTbReq->name, id);
continue;
}
code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, id);
if (code) {
tqError("vgId:%d failed to put sink tableInfo:%s into cache, s-task:%s", vgId, pCreateTbReq->name, id);
}
} }
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); tqDebug("s-task:%s build create table:%s msg complete", id, pCreateTbReq->name);
} }
reqs.nReqs = taosArrayGetSize(reqs.pArray); reqs.nReqs = taosArrayGetSize(reqs.pArray);
code = tqPutReqToQueue(pVnode, &reqs); code = tqPutReqToQueue(pVnode, &reqs);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqError("s-task:%s failed to send create table msg", pTask->id.idStr); tqError("s-task:%s failed to send create table msg", id);
} }
_end: _end:
@ -348,15 +403,26 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k); SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k);
if (pNewRow->ts < pOldRow->ts) { if (pNewRow->ts < pOldRow->ts) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1; j += 1;
} else if (pNewRow->ts > pOldRow->ts) { } else if (pNewRow->ts > pOldRow->ts) {
taosArrayPush(pFinal, &pOldRow); void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
} else { } else {
// check for the existance of primary key // check for the existance of primary key
if (pNewRow->numOfPKs == 0) { if (pNewRow->numOfPKs == 0) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
j += 1; j += 1;
tRowDestroy(pOldRow); tRowDestroy(pOldRow);
@ -369,7 +435,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
int32_t ret = tRowKeyCompare(&kNew, &kOld); int32_t ret = tRowKeyCompare(&kNew, &kOld);
if (ret <= 0) { if (ret <= 0) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1; j += 1;
if (ret == 0) { if (ret == 0) {
@ -377,7 +447,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
tRowDestroy(pOldRow); tRowDestroy(pOldRow);
} }
} else { } else {
taosArrayPush(pFinal, &pOldRow); void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
} }
} }
@ -386,12 +460,18 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
while (j < newLen) { while (j < newLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++); SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++);
taosArrayPush(pFinal, &pRow); void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
while (k < oldLen) { while (k < oldLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++); SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++);
taosArrayPush(pFinal, &pRow); void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
taosArrayDestroy(pNew->aRowP); taosArrayDestroy(pNew->aRowP);
@ -425,34 +505,40 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
return true; return true;
} }
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule) { SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq) {
*pReq = NULL;
SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
if (pCreateTbReq == NULL) { if (pCreateTbReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
taosArrayClear(pTagArray); taosArrayClear(pTagArray);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1); int32_t code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
if (code != 0) {
STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; return code;
taosArrayPush(pTagArray, &tagVal);
tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pCreateTbReq->ctb.tagName = createDefaultTagColName(); STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
void* p = taosArrayPush(pTagArray, &tagVal);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
return code;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
// set table name // set table name
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule); setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule);
return pCreateTbReq; *pReq = pCreateTbReq;
return code;
} }
int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) { int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) {
@ -555,7 +641,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
break; break;
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex);
if (colDataIsNull_s(pColData, j)) { if (colDataIsNull_s(pColData, j)) {
@ -566,7 +655,11 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
dataIndex++; dataIndex++;
} else { } else {
void* colData = colDataGetData(pColData, j); void* colData = colDataGetData(pColData, j);
@ -574,12 +667,18 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
SValue sv = SValue sv =
(SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SValue sv = {.type = pCol->type}; SValue sv = {.type = pCol->type};
memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
dataIndex++; dataIndex++;
} }
@ -596,7 +695,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
} }
ASSERT(pRow); ASSERT(pRow);
taosArrayPush(pTableData->aRowP, &pRow); void* p = taosArrayPush(pTableData->aRowP, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
taosArrayDestroy(pVals); taosArrayDestroy(pVals);
@ -665,6 +767,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
STableSinkInfo* pTableSinkInfo = NULL; STableSinkInfo* pTableSinkInfo = NULL;
int32_t code = 0;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo); bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo);
@ -686,7 +789,11 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} else { // this groupId has not been kept in cache yet } else { // this groupId has not been kept in cache yet
if (dstTableName[0] == 0) { if (dstTableName[0] == 0) {
memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); memset(dstTableName, 0, TSDB_TABLE_NAME_LEN);
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); code = buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
if (code) {
tqDebug("s-task:%s failed to build auto create table-name:%s, groupId:0x%" PRId64, id, dstTableName, groupId);
return code;
}
} else { } else {
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
@ -699,8 +806,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} }
} }
int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); if (code == 0) {
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName);
} else {
tqDebug("s-task:%s failed to build new sinkTableInfo, dstTable:%s", id, dstTableName);
return code;
}
} }
if (alreadyCached) { if (alreadyCached) {
@ -731,20 +843,20 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = code =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1),
&pTableData->pCreateTbReq);
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) { if (code) {
tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tstrerror(code));
tstrerror(terrno));
taosMemoryFree(pTableSinkInfo); taosMemoryFree(pTableSinkInfo);
return terrno; return code;
} }
pTableSinkInfo->uid = 0; pTableSinkInfo->uid = 0;
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} else { } else {
metaReaderClear(&mr); metaReaderClear(&mr);
@ -765,12 +877,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableSinkInfo->uid = mr.me.uid; pTableSinkInfo->uid = mr.me.uid;
metaReaderClear(&mr); metaReaderClear(&mr);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} }
} }
} }
return TDB_CODE_SUCCESS; return code;
} }
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
@ -864,14 +976,21 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) { if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq); tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id); (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id);
tbData.pCreateTbReq = NULL; tbData.pCreateTbReq = NULL;
} }
continue; continue;
} }
taosArrayPush(submitReq.aSubmitTbData, &tbData); void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqDebug("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
}
code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1);
if (code) { // failed and continue
tqDebug("vgId:%d, s-task:%s submit msg failed, data lost", vgId, id);
}
} }
} }
} else { } else {
@ -918,16 +1037,24 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) { if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq); tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id); (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id);
tbData.pCreateTbReq = NULL; tbData.pCreateTbReq = NULL;
} }
continue; continue;
} }
taosArrayPush(submitReq.aSubmitTbData, &tbData); void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqError("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
continue;
}
int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); code = taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
if (code) {
tqError("vgId:%d, s-task:%s failed to put group into index map, code:%s", vgId, id, tstrerror(code));
continue;
}
} else { } else {
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
@ -951,7 +1078,10 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
taosHashCleanup(pTableIndexMap); taosHashCleanup(pTableIndexMap);
if (hasSubmit) { if (hasSubmit) {
doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks);
if (code) { // failed and continue
tqError("vgId:%d failed to build and send submit msg", vgId);
}
} else { } else {
tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); tqDebug("vgId:%d, s-task:%s write results completed", vgId, id);
@ -989,7 +1119,11 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char
} }
int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId)); int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId));
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId); if (code == 0) {
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId);
} else {
tqError("s-task:%s failed to remove table meta from hashmap, groupId:%" PRId64, id, groupId);
}
return code; return code;
} }
@ -1019,10 +1153,14 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock*
void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
tEncodeSBatchDeleteReq(&encoder, &deleteReq); code = tEncodeSBatchDeleteReq(&encoder, &deleteReq);
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosArrayDestroy(deleteReq.deleteReqs); taosArrayDestroy(deleteReq.deleteReqs);
if (code) {
return code;
}
((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode); ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode);
SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)}; SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)};

64
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -37,7 +37,12 @@ int32_t tqScanWal(STQ* pTq) {
// check all tasks // check all tasks
int32_t numOfTasks = 0; int32_t numOfTasks = 0;
bool shouldIdle = true; bool shouldIdle = true;
doScanWalForAllTasks(pMeta, &shouldIdle);
int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle);
if (code) {
tqError("vgId:%d failed to start all tasks, try next time", vgId);
return code;
}
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
int32_t times = (--pMeta->scanInfo.scanCounter); int32_t times = (--pMeta->scanInfo.scanCounter);
@ -51,9 +56,13 @@ int32_t tqScanWal(STQ* pTq) {
if (times > 0) { if (times > 0) {
tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION);
tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION); code = tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION);
if (code) {
tqError("vgId:%d sched scan wal in %dms failed, ignore this failure", vgId, SCAN_WAL_IDLE_DURATION);
}
} }
return 0;
return code;
} }
typedef struct SBuildScanWalMsgParam { typedef struct SBuildScanWalMsgParam {
@ -69,28 +78,44 @@ static void doStartScanWal(void* param, void* tmrId) {
tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks, tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks,
pTq->pVnode->restored); pTq->pVnode->restored);
/*int32_t code = */ streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA);
taosMemoryFree(pParam); taosMemoryFree(pParam);
if (code) {
tqError("vgId:%d failed sched task to scan wal", vgId);
}
} }
int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam));
if (pParam == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pParam->pTq = pTq; pParam->pTq = pTq;
pParam->numOfTasks = numOfTasks; pParam->numOfTasks = numOfTasks;
tmr_h pTimer = streamTimerGetInstance(); tmr_h pTimer = NULL;
ASSERT(pTimer); code = streamTimerGetInstance(&pTimer);
if (code) {
tqError("vgId:%d failed to get tmr ctrl during sched scan wal", vgId);
return code;
}
if (pMeta->scanInfo.scanTimer == NULL) { if (pMeta->scanInfo.scanTimer == NULL) {
pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer);
} else { } else {
taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer);
if (code) {
tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration);
}
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
@ -207,7 +232,11 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) {
double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0; double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0;
qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs", qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs",
id, pTask->step2Range.minVer, maxVer, el); id, pTask->step2Range.minVer, maxVer, el);
/*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask); int32_t code = streamTaskPutTranstateIntoInputQ(pTask);
if (code) {
qError("s-task:%s failed to put trans-state into inputQ", id);
}
return true; return true;
} else { } else {
qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64 qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64
@ -290,8 +319,12 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems
break; break;
} }
} else { } else {
walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer);
code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
if (code) {
tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer);
}
break; break;
} }
} }
@ -347,25 +380,26 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX; int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name); tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems); bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if ((numOfItems > 0) || hasNewData) { if ((numOfItems > 0) || hasNewData) {
noDataInWal = false; noDataInWal = false;
code = streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
return -1; taosArrayDestroy(pTaskList);
return code;
} }
} }
@ -378,5 +412,5 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
} }
taosArrayDestroy(pTaskList); taosArrayDestroy(pTaskList);
return 0; return TSDB_CODE_SUCCESS;
} }

209
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -160,6 +160,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
bool updated = false; bool updated = false;
int32_t code = 0;
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
@ -258,26 +259,40 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
} }
} }
// save // stream do update the nodeEp info, write it into stream meta.
if (updated) { if (updated) {
tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId); tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId);
streamMetaSaveTask(pMeta, pTask); code = streamMetaSaveTask(pMeta, pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamMetaSaveTask(pMeta, *ppHTask); code = streamMetaSaveTask(pMeta, *ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code));
}
} }
} else { } else {
tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId); tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId);
} }
streamTaskStop(pTask); code = streamTaskStop(pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamTaskStop(*ppHTask); code = streamTaskStop(*ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code));
}
} }
// keep info // keep info
streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st); streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st);
rsp.code = 0; rsp.code = TSDB_CODE_SUCCESS;
// possibly only handle the stream task. // possibly only handle the stream task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
@ -305,13 +320,16 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
#if 0 #if 0
taosMSleep(5000);// for test purpose, to trigger the leader election taosMSleep(5000);// for test purpose, to trigger the leader election
#endif #endif
tqStreamTaskStartAsync(pMeta, cb, true); code = tqStreamTaskStartAsync(pMeta, cb, true);
if (code) {
tqError("vgId:%d async start all tasks, failed, code:%s", vgId, tstrerror(code));
}
} }
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return rsp.code; return rsp.code; // always return true
} }
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -333,7 +351,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask) { if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
return -1; return -1;
@ -393,14 +411,14 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask) { if (pTask && (code == 0)) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code); code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} else { } else {
tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId);
terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST;
return terrno; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
} }
@ -408,16 +426,22 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SDecoder decoder; SDecoder decoder;
SStreamRetrieveReq req; SStreamRetrieveReq req;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req); code = tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d failed to decode retrieve msg, quit handling it", pMeta->vgId);
return code;
}
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || code != 0) {
tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
req.dstTaskId); req.dstTaskId);
tCleanupStreamRetrieveReq(&req); tCleanupStreamRetrieveReq(&req);
@ -446,6 +470,7 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamTaskCheckReq req; SStreamTaskCheckReq req;
SStreamTaskCheckRsp rsp = {0}; SStreamTaskCheckRsp rsp = {0};
@ -453,9 +478,14 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamTaskCheckReq(&decoder, &req); code = tDecodeStreamTaskCheckReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d decode check msg failed, not handle this msg", pMeta->vgId);
return code;
}
streamTaskProcessCheckMsg(pMeta, &req, &rsp); streamTaskProcessCheckMsg(pMeta, &req, &rsp);
return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId); return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId);
} }
@ -490,7 +520,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask); code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask);
if (pTask == NULL) { if ((pTask == NULL) || (code != 0)) {
return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId);
} }
@ -518,19 +548,25 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask);
if (pTask == NULL) { if (code != 0) {
tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId,
pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
if (code) {
return code;
}
{ // send checkpoint ready rsp { // send checkpoint ready rsp
SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg)); SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg));
if (pReadyRsp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReadyRsp->upstreamTaskId = req.upstreamTaskId; pReadyRsp->upstreamTaskId = req.upstreamTaskId;
pReadyRsp->upstreamNodeId = req.upstreamNodeId; pReadyRsp->upstreamNodeId = req.upstreamNodeId;
@ -606,8 +642,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
if (restored) { if (restored) {
SStreamTask* p = NULL; SStreamTask* p = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &p); code = streamMetaAcquireTask(pMeta, streamId, taskId, &p);
if ((p != NULL) && (p->info.fillHistory == 0)) { if ((p != NULL) && (code == 0) && (p->info.fillHistory == 0)) {
tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); code = tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
} }
if (p != NULL) { if (p != NULL) {
@ -631,6 +667,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
STaskId hTaskId = {0}; STaskId hTaskId = {0};
tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
@ -649,8 +686,12 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamTaskSetRemoveBackendFiles(pTask); streamTaskSetRemoveBackendFiles(pTask);
streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
if (code) {
tqError("s-task:0x%x failed to clear related fill-history info, still exists", pReq->taskId);
}
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -658,11 +699,17 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
// drop the related fill-history task firstly // drop the related fill-history task firstly
if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
}
} }
// drop the stream task now // drop the stream task now
streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); code = streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop task failed", pReq->taskId, vgId);
}
// commit the update // commit the update
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
@ -674,12 +721,13 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; return 0; // always return success
} }
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId);
@ -689,7 +737,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask != NULL && (*ppTask) != NULL) { if (ppTask != NULL && (*ppTask) != NULL) {
streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq); code = streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq);
} else { // failed to get the task. } else { // failed to get the task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
tqError( tqError(
@ -700,7 +748,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// always return success when handling the requirement issued by mnode during transaction. // always return success when handling the requirement issued by mnode during transaction.
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
@ -746,7 +794,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
if (isLeader && !tsDisableStream) { if (isLeader && !tsDisableStream) {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
streamMetaStartAllTasks(pMeta); code = streamMetaStartAllTasks(pMeta);
} else { } else {
streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId);
pMeta->startInfo.restartCount = 0; pMeta->startInfo.restartCount = 0;
@ -765,16 +813,16 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) { if (type == STREAM_EXEC_T_START_ONE_TASK) {
streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); (void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0; return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) { } else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
streamMetaStartAllTasks(pMeta); (void) streamMetaStartAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
restartStreamTasks(pMeta, isLeader); (void) restartStreamTasks(pMeta, isLeader);
return 0; return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
streamMetaStopAllTasks(pMeta); (void) streamMetaStopAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
@ -783,7 +831,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { if (pTask != NULL && (code == 0)) {
char* pStatus = NULL; char* pStatus = NULL;
if (streamTaskReadyToRun(pTask, &pStatus)) { if (streamTaskReadyToRun(pTask, &pStatus)) {
int64_t execTs = pTask->status.lastExecTs; int64_t execTs = pTask->status.lastExecTs;
@ -804,12 +852,12 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { // even in halt status, the data in inputQ must be processed if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL; char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) { if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer); pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
streamExecTask(pTask); (void) streamExecTask(pTask);
} else { } else {
int8_t status = streamTaskSetSchedStatusInactive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -829,6 +877,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskStartInfo* pStartInfo = &pMeta->startInfo;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
bool scanWal = false; bool scanWal = false;
int32_t code = 0;
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
if (pStartInfo->startAllTasks == 1) { if (pStartInfo->startAllTasks == 1) {
@ -844,8 +893,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
pStartInfo->restartCount); pStartInfo->restartCount);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); return restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER));
return TSDB_CODE_SUCCESS;
} else { } else {
if (pStartInfo->restartCount == 0) { if (pStartInfo->restartCount == 0) {
tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId); tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId);
@ -862,10 +910,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
if (scanWal && (vgId != SNODE_HANDLE)) { if (scanWal && (vgId != SNODE_HANDLE)) {
tqDebug("vgId:%d start scan wal for executing tasks", vgId); tqDebug("vgId:%d start scan wal for executing tasks", vgId);
tqScanWalAsync(pMeta->ahandle, true); code = tqScanWalAsync(pMeta->ahandle, true);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
@ -873,7 +921,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -881,7 +929,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
streamTaskClearCheckInfo(pTask, true); streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks // clear flag set during do checkpoint, and open inputQ for all upstream tasks
@ -904,7 +952,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -915,7 +963,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId); pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
@ -929,11 +977,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
pTask->id.idStr, (int32_t)pReq->downstreamTaskId); pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS); TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return code;
return TSDB_CODE_SUCCESS;
} }
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
@ -948,7 +995,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
// re-send the lost checkpoint-trigger msg to downstream task // re-send the lost checkpoint-trigger msg to downstream task
tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
(int32_t)pReq->downstreamTaskId, checkpointId, transId); (int32_t)pReq->downstreamTaskId, checkpointId, transId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_SUCCESS); TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait } else { // not send checkpoint-trigger yet, wait
int32_t recv = 0, total = 0; int32_t recv = 0, total = 0;
@ -962,7 +1009,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"sending checkpoint-source/trigger", "sending checkpoint-source/trigger",
pTask->id.idStr, recv, total); pTask->id.idStr, recv, total);
} }
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
} else { // upstream not recv the checkpoint-source/trigger till now } else { // upstream not recv the checkpoint-source/trigger till now
@ -971,12 +1018,12 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger", "upstream sending checkpoint-source/trigger",
pTask->id.idStr); pTask->id.idStr);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -984,19 +1031,19 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError( tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pRsp->taskId); pMeta->vgId, pRsp->taskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d", tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
@ -1004,7 +1051,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId); pReq->taskId);
// since task is in [STOP|DROPPING] state, it is safe to assume the pause is active // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active
@ -1018,7 +1065,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pHistoryTask = NULL; pHistoryTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask);
if (pHistoryTask == NULL) { if (pHistoryTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
", it may have been dropped already", ", it may have been dropped already",
pMeta->vgId, pTask->hTaskInfo.id.taskId); pMeta->vgId, pTask->hTaskInfo.id.taskId);
@ -1042,6 +1089,8 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
bool fromVnode) { bool fromVnode) {
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
int32_t code = 0;
if (pTask == NULL) { if (pTask == NULL) {
return -1; return -1;
} }
@ -1065,18 +1114,18 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
pTask->hTaskInfo.operatorOpen = false; pTask->hTaskInfo.operatorOpen = false;
streamStartScanHistoryAsync(pTask, igUntreated); code = streamStartScanHistoryAsync(pTask, igUntreated);
} else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
tqScanWalAsync((STQ*)handle, false); code = tqScanWalAsync((STQ*)handle, false);
} else { } else {
streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
} }
} /*else { } /*else {
ASSERT(status != TASK_STATUS__UNINIT); ASSERT(status != TASK_STATUS__UNINIT);
}*/ }*/
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return 0; return code;
} }
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) { int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) {
@ -1086,15 +1135,15 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name); tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
if (code != 0) { if (code != 0) {
@ -1104,11 +1153,11 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL; SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask) { if (pHTask && (code == 0)) {
taosThreadMutexLock(&pHTask->lock); streamMutexLock(&pHTask->lock);
SStreamTaskState p = streamTaskGetStatus(pHTask); SStreamTaskState p = streamTaskGetStatus(pHTask);
tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name); tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name);
taosThreadMutexUnlock(&pHTask->lock); streamMutexUnlock(&pHTask->lock);
code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode);
} }
@ -1139,15 +1188,15 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId); pRsp->downstreamNodeId, pRsp->downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); code = streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -1173,11 +1222,11 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId); pMeta->vgId, req.taskId);
streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); (void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return TSDB_CODE_SUCCESS; return code;
} }
// discard the rsp, since it is expired. // discard the rsp, since it is expired.
@ -1193,13 +1242,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode", tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode",
pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId); pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) { if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId); pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1215,14 +1264,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
} }
pTask->chkInfo.consensusTransId = req.transId; pTask->chkInfo.consensusTransId = req.transId;
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if (pMeta->role == NODE_ROLE_LEADER) { if (pMeta->role == NODE_ROLE_LEADER) {
/*code = */ tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId);
} else { } else {
tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr); tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }

54
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -294,6 +294,14 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree(p->pCurrSchema); taosMemoryFree(p->pCurrSchema);
if (p->rowKey.numOfPKs > 0) {
for (int32_t i = 0; i < p->rowKey.numOfPKs; i++) {
if (IS_VAR_DATA_TYPE(p->rowKey.pks[i].type)) {
taosMemoryFree(p->rowKey.pks[i].pData);
}
}
}
if (p->pLDataIterArray) { if (p->pLDataIterArray) {
destroySttBlockReader(p->pLDataIterArray, NULL); destroySttBlockReader(p->pLDataIterArray, NULL);
} }
@ -325,13 +333,26 @@ void* tsdbCacherowsReaderClose(void* pReader) {
return NULL; return NULL;
} }
static void freeItem(void* pItem) { static void freeItemOfRow(void* pItem) {
SLastCol* pCol = (SLastCol*)pItem; SLastCol* pCol = (SLastCol*)pItem;
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) { if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData); taosMemoryFree(pCol->colVal.value.pData);
} }
} }
static void freeItemWithPk(void* pItem) {
SLastCol* pCol = (SLastCol*)pItem;
for (int i = 0; i < pCol->rowKey.numOfPKs; i++) {
if (IS_VAR_DATA_TYPE(pCol->rowKey.pks[i].type)) {
taosMemoryFree(pCol->rowKey.pks[i].pData);
}
}
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData);
}
}
static int32_t tsdbCacheQueryReseek(void* pQHandle) { static int32_t tsdbCacheQueryReseek(void* pQHandle) {
int32_t code = 0; int32_t code = 0;
SCacheRowsReader* pReader = pQHandle; SCacheRowsReader* pReader = pQHandle;
@ -407,6 +428,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
struct STColumn* pCol = &pr->pSchema->columns[slotId]; struct STColumn* pCol = &pr->pSchema->columns[slotId];
SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
if (pr->rowKey.numOfPKs > 0) {
p.rowKey.numOfPKs = pr->rowKey.numOfPKs;
for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) {
p.rowKey.pks[j].type = pr->pkColumn.type;
if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) {
p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes);
}
}
}
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
} }
@ -420,7 +451,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
continue; continue;
} }
@ -432,7 +463,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
SLastCol* p = taosArrayGet(pLastCols, k); SLastCol* p = taosArrayGet(pLastCols, k);
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
if (pColVal->rowKey.ts > p->rowKey.ts) { if (tRowKeyCompare(&pColVal->rowKey, &p->rowKey) > 0) {
if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) {
if (!COL_VAL_IS_VALUE(&p->colVal)) { if (!COL_VAL_IS_VALUE(&p->colVal)) {
hasNotNullRow = false; hasNotNullRow = false;
@ -445,6 +476,15 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
hasRes = true; hasRes = true;
p->rowKey.ts = pColVal->rowKey.ts; p->rowKey.ts = pColVal->rowKey.ts;
for (int32_t j = 0; j < p->rowKey.numOfPKs; j++) {
if (IS_VAR_DATA_TYPE(p->rowKey.pks[j].type)) {
memcpy(p->rowKey.pks[j].pData, pColVal->rowKey.pks[j].pData, pColVal->rowKey.pks[j].nData);
p->rowKey.pks[j].nData = pColVal->rowKey.pks[j].nData;
} else {
p->rowKey.pks[j].val = pColVal->rowKey.pks[j].val;
}
}
if (k == 0) { if (k == 0) {
if (TARRAY_SIZE(pTableUidList) == 0) { if (TARRAY_SIZE(pTableUidList) == 0) {
taosArrayPush(pTableUidList, &uid); taosArrayPush(pTableUidList, &uid);
@ -483,26 +523,26 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
} }
} }
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
} }
if (hasRes) { if (hasRes) {
saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
} }
taosArrayDestroyEx(pLastCols, freeItem); taosArrayDestroyEx(pLastCols, freeItemWithPk);
} else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) {
tb_uid_t uid = pTableList[i].uid; tb_uid_t uid = pTableList[i].uid;
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
continue; continue;
} }
saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
taosArrayPush(pTableUidList, &uid); taosArrayPush(pTableUidList, &uid);

29
source/dnode/vnode/src/vnd/vnodeAsync.c
View File

@ -190,10 +190,10 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) {
task->prev->next = task->next; task->prev->next = task->next;
task->next->prev = task->prev; task->next->prev = task->prev;
if (task->cancel) { if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){ TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel, .cancel = task->cancel,
.arg = task->arg, .arg = task->arg,
}); }));
} }
vnodeAsyncTaskDone(async, task); vnodeAsyncTaskDone(async, task);
} }
@ -206,6 +206,9 @@ static void *vnodeAsyncLoop(void *arg) {
SVWorker *worker = (SVWorker *)arg; SVWorker *worker = (SVWorker *)arg;
SVAsync *async = worker->async; SVAsync *async = worker->async;
SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
if (cancelArray == NULL) {
return NULL;
}
setThreadName(async->label); setThreadName(async->label);
@ -466,7 +469,7 @@ int32_t vnodeAsyncOpen(int32_t numOfThreads) {
vnodeAsyncSetWorkers(2, numOfThreads); vnodeAsyncSetWorkers(2, numOfThreads);
_exit: _exit:
return 0; return code;
} }
int32_t vnodeAsyncClose() { int32_t vnodeAsyncClose() {
@ -748,10 +751,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
task->prev->next = task->next; task->prev->next = task->next;
task->next->prev = task->prev; task->next->prev = task->prev;
if (task->cancel) { if (task->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){ TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = task->cancel, .cancel = task->cancel,
.arg = task->arg, .arg = task->arg,
}); }));
} }
vnodeAsyncTaskDone(async, task); vnodeAsyncTaskDone(async, task);
} }
@ -763,10 +766,10 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
channel->scheduled->prev->next = channel->scheduled->next; channel->scheduled->prev->next = channel->scheduled->next;
channel->scheduled->next->prev = channel->scheduled->prev; channel->scheduled->next->prev = channel->scheduled->prev;
if (channel->scheduled->cancel) { if (channel->scheduled->cancel) {
taosArrayPush(cancelArray, &(SVATaskCancelInfo){ TAOS_UNUSED(taosArrayPush(cancelArray, &(SVATaskCancelInfo){
.cancel = channel->scheduled->cancel, .cancel = channel->scheduled->cancel,
.arg = channel->scheduled->arg, .arg = channel->scheduled->arg,
}); }));
} }
vnodeAsyncTaskDone(async, channel->scheduled); vnodeAsyncTaskDone(async, channel->scheduled);
} }

22
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -16,13 +16,12 @@
#include "vnd.h" #include "vnd.h"
/* ------------------------ STRUCTURES ------------------------ */ /* ------------------------ STRUCTURES ------------------------ */
static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { static int32_t vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) {
SVBufPool *pPool; SVBufPool *pPool;
pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); pPool = taosMemoryMalloc(sizeof(SVBufPool) + size);
if (pPool == NULL) { if (pPool == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
memset(pPool, 0, sizeof(SVBufPool)); memset(pPool, 0, sizeof(SVBufPool));
@ -44,14 +43,12 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPoo
pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock));
if (!pPool->lock) { if (!pPool->lock) {
taosMemoryFree(pPool); taosMemoryFree(pPool);
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
if (taosThreadSpinInit(pPool->lock, 0) != 0) { if (taosThreadSpinInit(pPool->lock, 0) != 0) {
taosMemoryFree((void *)pPool->lock); taosMemoryFree((void *)pPool->lock);
taosMemoryFree(pPool); taosMemoryFree(pPool);
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
} else { } else {
pPool->lock = NULL; pPool->lock = NULL;
@ -77,10 +74,11 @@ int vnodeOpenBufPool(SVnode *pVnode) {
for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) {
// create pool // create pool
if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) { int32_t code;
if ((code = vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i]))) {
vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
vnodeCloseBufPool(pVnode); vnodeCloseBufPool(pVnode);
return -1; return code;
} }
// add to free list // add to free list
@ -274,8 +272,6 @@ _exit:
} }
int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
int32_t code = 0;
taosThreadMutexLock(&pPool->mutex); taosThreadMutexLock(&pPool->mutex);
pQNode->pNext = pPool->qList.pNext; pQNode->pNext = pPool->qList.pNext;
@ -285,9 +281,7 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
pPool->nQuery++; pPool->nQuery++;
taosThreadMutexUnlock(&pPool->mutex); taosThreadMutexUnlock(&pPool->mutex);
return 0;
_exit:
return code;
} }
void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) { void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) {

259
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -88,33 +88,33 @@ const ESyncRole vnodeStrToRole(char *str) {
int vnodeEncodeConfig(const void *pObj, SJson *pJson) { int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; const SVnodeCfg *pCfg = (SVnodeCfg *)pObj;
if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId));
if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddStringToObject(pJson, "dbname", pCfg->dbname));
if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId));
if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage));
if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache));
if (tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast));
if (tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize));
if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf));
if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap));
if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak));
if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma));
if (tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma));
if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision));
if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update));
if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression));
if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel));
if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days));
if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows));
if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows));
if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0));
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1));
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2));
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset));
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize));
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal));
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact));
if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize));
if (pCfg->tsdbCfg.retentions[0].keep > 0) { if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1; int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) { if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -124,61 +124,67 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
} }
} }
SJson *pNodeRetentions = tjsonCreateArray(); SJson *pNodeRetentions = tjsonCreateArray();
if (pNodeRetentions == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tjsonAddItemToObject(pJson, "retentions", pNodeRetentions); tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
for (int32_t i = 0; i < nRetention; ++i) { for (int32_t i = 0; i < nRetention; ++i) {
SJson *pNodeRetention = tjsonCreateObject(); SJson *pNodeRetention = tjsonCreateObject();
const SRetention *pRetention = pCfg->tsdbCfg.retentions + i; const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq));
tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit));
tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep));
tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit));
tjsonAddItemToArray(pNodeRetentions, pNodeRetention); TAOS_CHECK_RETURN(tjsonAddItemToArray(pNodeRetentions, pNodeRetention));
} }
} }
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm));
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId));
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod));
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod));
if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod));
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize));
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize));
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level));
if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles));
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm));
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm));
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger));
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin));
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd));
if (tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange));
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod));
if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix));
if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum));
if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex));
if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion));
if (tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables));
if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables));
if (tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries));
if (tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries));
if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1;
SJson *nodeInfo = tjsonCreateArray(); SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1; if (nodeInfo == NULL) {
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; return TSDB_CODE_OUT_OF_MEMORY;
}
TAOS_CHECK_RETURN(tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo));
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId, vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion); pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
SJson *info = tjsonCreateObject(); SJson *info = tjsonCreateObject();
if (info == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort));
if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn));
if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId));
if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId));
if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)));
if (tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddItemToArray(nodeInfo, info));
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId); pNode->nodeId);
} }
@ -191,50 +197,50 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code); tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code);
if (code < 0) return -1; if (code) return code;
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1; if ((code = tjsonGetStringValue(pJson, "dbname", pCfg->dbname))) return code;
tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code); tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code); tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code); tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code); tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code); tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code); tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code); tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code); tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code); tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code); tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code); tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code); tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code); tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code); tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code); tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code); tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code); tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code); tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code); tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code); tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code); tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code);
if (code < 0) return -1; if (code) return code;
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
int32_t nRetention = tjsonGetArraySize(pNodeRetentions); int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
if (nRetention > TSDB_RETENTION_MAX) { if (nRetention > TSDB_RETENTION_MAX) {
@ -244,52 +250,54 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i);
ASSERT(pNodeRetention != NULL); ASSERT(pNodeRetention != NULL);
tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code); tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code); tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code); tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
if (code) return code;
} }
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code); tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1; if (code) return code;
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) { if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) { if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
} else { } else {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
} }
} }
#endif #endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code); tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code); tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code); tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code); tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code); tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code); tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code); tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code); tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1; if (code) return code;
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) { if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) { if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
} else { } else {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
} }
} }
#endif #endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code); tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1; if (code) return code;
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) { if (tsEncryptKey[0] == 0) {
@ -303,35 +311,35 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER; if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code); tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashChange", pCfg->hashChange, code); tjsonGetNumberValue(pJson, "hashChange", pCfg->hashChange, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code); tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code); tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code);
if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code); tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code);
if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code); tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion, code); tjsonGetNumberValue(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables, code); tjsonGetNumberValue(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables, code); tjsonGetNumberValue(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables, code); tjsonGetNumberValue(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries, code); tjsonGetNumberValue(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code); tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code);
if (code < 0) return -1; if (code) return code;
SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(nodeInfo); int arraySize = tjsonGetArraySize(nodeInfo);
@ -344,15 +352,15 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1; if (info == NULL) return -1;
tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code); tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code);
if (code < 0) return -1; if (code) return code;
tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn); tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn);
tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code); tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code);
if (code < 0) return -1; if (code) return code;
char role[10] = {0}; char role[10] = {0};
code = tjsonGetStringValue(info, "isReplica", role); code = tjsonGetStringValue(info, "isReplica", role);
if (code < 0) return -1; if (code) return code;
if (strlen(role) != 0) { if (strlen(role) != 0) {
pNode->nodeRole = vnodeStrToRole(role); pNode->nodeRole = vnodeStrToRole(role);
} else { } else {
@ -394,8 +402,7 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) {
} }
if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) { if (hashValue < pVnode->config.hashBegin || hashValue > pVnode->config.hashEnd) {
terrno = TSDB_CODE_VND_HASH_MISMATCH; return terrno = TSDB_CODE_VND_HASH_MISMATCH;
return -1;
} }
return 0; return 0;

200
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -125,20 +125,16 @@ int vnodeBegin(SVnode *pVnode) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// begin meta // begin meta
if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) { code = metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
// begin tsdb // begin tsdb
if (tsdbBegin(pVnode->pTsdb) < 0) { code = tsdbBegin(pVnode->pTsdb);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
// begin sma // begin sma
if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) { if (VND_IS_RSMA(pVnode)) {
code = terrno; code = smaBegin(pVnode->pSma);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -171,54 +167,41 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
} }
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile; TdFilePtr pFile = NULL;
char *data; char *data = NULL;
snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP); snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
// encode info code = vnodeEncodeInfo(pInfo, &data);
data = NULL; TSDB_CHECK_CODE(code, lino, _exit);
if (vnodeEncodeInfo(pInfo, &data) < 0) {
vError("failed to encode json info.");
return -1;
}
// save info to a vnode_tmp.json // save info to a vnode_tmp.json
pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) { if (pFile == NULL) {
vError("failed to open info file:%s for write:%s", fname, terrstr()); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
if (taosWriteFile(pFile, data, strlen(data)) < 0) { if (taosWriteFile(pFile, data, strlen(data)) < 0) {
vError("failed to write info file:%s error:%s", fname, terrstr()); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
} }
if (taosFsyncFile(pFile) < 0) { if (taosFsyncFile(pFile) < 0) {
vError("failed to fsync info file:%s error:%s", fname, terrstr()); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
} }
taosCloseFile(&pFile); _exit:
if (code) {
// free info binary vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
taosMemoryFree(data); } else {
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname,
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname, pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion);
pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion); }
return 0;
_err:
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(data); taosMemoryFree(data);
return -1; return code;
} }
int vnodeCommitInfo(const char *dir) { int vnodeCommitInfo(const char *dir) {
@ -229,8 +212,7 @@ int vnodeCommitInfo(const char *dir) {
snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP); snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
if (taosRenameFile(tfname, fname) < 0) { if (taosRenameFile(tfname, fname) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
vInfo("vnode info is committed, dir:%s", dir); vInfo("vnode info is committed, dir:%s", dir);
@ -238,6 +220,8 @@ int vnodeCommitInfo(const char *dir) {
} }
int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) { int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char *pData = NULL; char *pData = NULL;
@ -248,44 +232,35 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
// read info // read info
pFile = taosOpenFile(fname, TD_FILE_READ); pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
return -1;
} }
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _err;
} }
pData = taosMemoryMalloc(size + 1); pData = taosMemoryMalloc(size + 1);
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
if (taosReadFile(pFile, pData, size) < 0) { if (taosReadFile(pFile, pData, size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _err;
} }
pData[size] = '\0'; pData[size] = '\0';
taosCloseFile(&pFile);
// decode info // decode info
if (vnodeDecodeInfo(pData, pInfo) < 0) { code = vnodeDecodeInfo(pData, pInfo);
taosMemoryFree(pData); TSDB_CHECK_CODE(code, lino, _exit);
return -1;
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
} }
taosMemoryFree(pData); taosMemoryFree(pData);
return 0;
_err:
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(pData); return code;
return -1;
} }
static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
@ -297,7 +272,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
// wait last commit task // wait last commit task
vnodeAWait(&pVnode->commitTask); vnodeAWait(&pVnode->commitTask);
if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit; code = syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg);
TSDB_CHECK_CODE(code, lino, _exit);
pVnode->state.commitTerm = pVnode->state.applyTerm; pVnode->state.commitTerm = pVnode->state.applyTerm;
@ -312,17 +288,16 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN);
vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode)); vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode));
if (vnodeSaveInfo(dir, &pInfo->info) < 0) { code = vnodeSaveInfo(dir, &pInfo->info);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdbPreCommit(pVnode->pTsdb); tsdbPreCommit(pVnode->pTsdb);
metaPrepareAsyncCommit(pVnode->pMeta); code = metaPrepareAsyncCommit(pVnode->pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
code = smaPrepareAsyncCommit(pVnode->pSma); code = smaPrepareAsyncCommit(pVnode->pSma);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pVnode->mutex); taosThreadMutexLock(&pVnode->mutex);
ASSERT(pVnode->onCommit == NULL); ASSERT(pVnode->onCommit == NULL);
@ -459,10 +434,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
} }
// commit info // commit info
if (vnodeCommitInfo(dir) < 0) { code = vnodeCommitInfo(dir);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbCommitCommit(pVnode->pTsdb); code = tsdbCommitCommit(pVnode->pTsdb);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -472,10 +445,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) { code = metaFinishCommit(pVnode->pMeta, pInfo->txn);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
pVnode->state.committed = pInfo->info.state.committed; pVnode->state.committed = pInfo->info.state.committed;
@ -492,7 +463,7 @@ _exit:
} else { } else {
vInfo("vgId:%d, commit end", TD_VID(pVnode)); vInfo("vgId:%d, commit end", TD_VID(pVnode));
} }
return 0; return code;
} }
bool vnodeShouldRollback(SVnode *pVnode) { bool vnodeShouldRollback(SVnode *pVnode) {
@ -514,15 +485,15 @@ void vnodeRollback(SVnode *pVnode) {
offset = strlen(tFName); offset = strlen(tFName);
snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP);
(void)taosRemoveFile(tFName); TAOS_UNUSED(taosRemoveFile(tFName));
} }
static int vnodeEncodeState(const void *pObj, SJson *pJson) { static int vnodeEncodeState(const void *pObj, SJson *pJson) {
const SVState *pState = (SVState *)pObj; const SVState *pState = (SVState *)pObj;
if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit version", pState->committed));
if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID));
if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm));
return 0; return 0;
} }
@ -532,70 +503,67 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, "commit version", pState->committed, code); tjsonGetNumberValue(pJson, "commit version", pState->committed, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code); tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code); tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
if (code < 0) return -1; if (code) return code;
return 0; return 0;
} }
static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) { static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
SJson *pJson; int32_t code = 0;
char *pData; int32_t lino;
SJson *pJson = NULL;
*ppData = NULL; char *pData = NULL;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) { if (pJson == NULL) {
return -1; TSDB_CHECK_CODE(code = terrno, lino, _exit);
} }
if (tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config) < 0) { code = tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
if (tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state) < 0) { code = tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
pData = tjsonToString(pJson); pData = tjsonToString(pJson);
if (pData == NULL) { if (pData == NULL) {
goto _err; TSDB_CHECK_CODE(code = terrno, lino, _exit);
} }
tjsonDelete(pJson); tjsonDelete(pJson);
*ppData = pData; _exit:
return 0; if (code) {
tjsonDelete(pJson);
_err: *ppData = NULL;
tjsonDelete(pJson); } else {
return -1; *ppData = pData;
}
return code;
} }
int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) { int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
SJson *pJson = NULL; int32_t code = 0;
int32_t lino;
SJson *pJson = NULL;
pJson = tjsonParse(pData); pJson = tjsonParse(pData);
if (pJson == NULL) { if (pJson == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
return -1; return -1;
} }
if (tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config) < 0) { code = tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
if (tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state) < 0) { code = tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
tjsonDelete(pJson); tjsonDelete(pJson);
return code;
return 0;
_err:
tjsonDelete(pJson);
return -1;
} }

8
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -129,17 +129,17 @@ void initTqAPI(SStoreTqReader* pTq) {
pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable;
pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed;
pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it
// pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it // pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it
pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it
pTq->tqGetResultBlock = tqGetResultBlock; pTq->tqGetResultBlock = tqGetResultBlock;
// pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; // pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
pTq->tqGetResultBlockTime = tqGetResultBlockTime; pTq->tqGetResultBlockTime = tqGetResultBlockTime;
pTq->tqGetStreamExecProgress = tqGetStreamExecInfo; pTq->tqGetStreamExecProgress = tqGetStreamExecInfo;
} }
void initStateStoreAPI(SStateStore* pStore) { void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateInit = streamFileStateInit; pStore->streamFileStateInit = streamFileStateInit;

20
source/dnode/vnode/src/vnd/vnodeModule.c
View File

@ -19,31 +19,19 @@
static volatile int32_t VINIT = 0; static volatile int32_t VINIT = 0;
int vnodeInit(int nthreads) { int vnodeInit(int nthreads) {
int32_t init; if (atomic_val_compare_exchange_32(&VINIT, 0, 1)) {
init = atomic_val_compare_exchange_32(&VINIT, 0, 1);
if (init) {
return 0; return 0;
} }
if (vnodeAsyncOpen(nthreads) != 0) { TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads));
return -1; TAOS_CHECK_RETURN(walInit());
}
if (walInit() < 0) {
return -1;
}
return 0; return 0;
} }
void vnodeCleanup() { void vnodeCleanup() {
int32_t init = atomic_val_compare_exchange_32(&VINIT, 1, 0); if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return;
if (init == 0) return;
// set stop
vnodeAsyncClose(); vnodeAsyncClose();
walCleanUp(); walCleanUp();
smaCleanUp(); smaCleanUp();
} }

431
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -17,20 +17,19 @@
#include "vnd.h" #include "vnd.h"
static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) {
int32_t code = -1; int32_t code = 0;
STsdbFSetPartList *pList = tsdbFSetPartListCreate(); STsdbFSetPartList *pList = tsdbFSetPartListCreate();
if (pList == NULL) { if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _out; goto _out;
} }
if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) {
terrno = TSDB_CODE_INVALID_DATA_FMT; code = tDeserializeTsdbFSetPartList(buf, bufLen, pList);
goto _out; if (code) goto _out;
}
if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) { code = tsdbFSetPartListToRangeDiff(pList, ppRanges);
goto _out; if (code) goto _out;
}
code = 0;
_out: _out:
tsdbFSetPartListDestroy(&pList); tsdbFSetPartListDestroy(&pList);
return code; return code;
@ -48,29 +47,29 @@ struct SVSnapReader {
int8_t metaDone; int8_t metaDone;
SMetaSnapReader *pMetaReader; SMetaSnapReader *pMetaReader;
// tsdb // tsdb
int8_t tsdbDone; int8_t tsdbDone;
TFileSetRangeArray *pRanges; TFileSetRangeArray *pRanges;
STsdbSnapReader *pTsdbReader; STsdbSnapReader *pTsdbReader;
// tsdb raw // tsdb raw
int8_t tsdbRAWDone; int8_t tsdbRAWDone;
STsdbSnapRAWReader *pTsdbRAWReader; STsdbSnapRAWReader *pTsdbRAWReader;
// tq // tq
int8_t tqHandleDone; int8_t tqHandleDone;
STqSnapReader *pTqSnapReader; STqSnapReader *pTqSnapReader;
int8_t tqOffsetDone; int8_t tqOffsetDone;
STqSnapReader *pTqOffsetReader; STqSnapReader *pTqOffsetReader;
int8_t tqCheckInfoDone; int8_t tqCheckInfoDone;
STqSnapReader *pTqCheckInfoReader; STqSnapReader *pTqCheckInfoReader;
// stream // stream
int8_t streamTaskDone; int8_t streamTaskDone;
SStreamTaskReader *pStreamTaskReader; SStreamTaskReader *pStreamTaskReader;
int8_t streamStateDone; int8_t streamStateDone;
SStreamStateReader *pStreamStateReader; SStreamStateReader *pStreamStateReader;
// rsma // rsma
int8_t rsmaDone; int8_t rsmaDone;
TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
SRSmaSnapReader *pRsmaReader; SRSmaSnapReader *pRsmaReader;
}; };
static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
@ -88,14 +87,15 @@ static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader,
} }
static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
int32_t code = 0;
SVnode *pVnode = pReader->pVnode; SVnode *pVnode = pReader->pVnode;
int32_t code = -1;
if (pParam->data) { if (pParam->data) {
// decode // decode
SSyncTLV *datHead = (void *)pParam->data; SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT; code = TSDB_CODE_INVALID_DATA_FMT;
terrno = code;
goto _out; goto _out;
} }
@ -116,21 +116,25 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
if (ppRanges == NULL) { if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out; goto _out;
} }
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
if (code) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
if (code) {
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
} break; } break;
default: default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out; goto _out;
} }
} }
@ -147,7 +151,7 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode), vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode),
(pReader->tsdbDone ? "raw" : "normal")); (pReader->tsdbDone ? "raw" : "normal"));
} }
code = 0;
_out: _out:
return code; return code;
} }
@ -160,42 +164,43 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader)); pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) { if (pReader == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
pReader->pVnode = pVnode; pReader->pVnode = pVnode;
pReader->sver = sver; pReader->sver = sver;
pReader->ever = ever; pReader->ever = ever;
// snapshot info // snapshot info
if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) { code = vnodeSnapReaderDealWithSnapInfo(pReader, pParam);
goto _err; if (code) goto _exit;
}
// open tsdb snapshot raw reader // open tsdb snapshot raw reader
if (!pReader->tsdbRAWDone) { if (!pReader->tsdbRAWDone) {
ASSERT(pReader->sver == 0); ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err; if (code) goto _exit;
} }
// check snapshot ever // check snapshot ever
SSnapshot snapshot = {0}; SSnapshot snapshot = {0};
vnodeGetSnapshot(pVnode, &snapshot); code = vnodeGetSnapshot(pVnode, &snapshot);
if (code) goto _exit;
if (ever != snapshot.lastApplyIndex) { if (ever != snapshot.lastApplyIndex) {
vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64, vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64,
TD_VID(pVnode), ever, snapshot.lastApplyIndex); TD_VID(pVnode), ever, snapshot.lastApplyIndex);
code = TSDB_CODE_SYN_INTERNAL_ERROR; code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err; goto _exit;
} }
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); _exit:
*ppReader = pReader; if (code) {
return code; vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
*ppReader = NULL;
_err: } else {
vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code)); vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = NULL; *ppReader = pReader;
}
return code; return code;
} }
@ -245,6 +250,7 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVnode *pVnode = pReader->pVnode; SVnode *pVnode = pReader->pVnode;
int32_t vgId = TD_VID(pReader->pVnode); int32_t vgId = TD_VID(pReader->pVnode);
@ -261,31 +267,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ);
if (NULL == pFile) { if (NULL == pFile) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
int64_t size; int64_t size;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto _err; TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
} }
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto _err; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
} }
((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG; ((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG;
((SSnapDataHdr *)(*ppData))->size = size + 1; ((SSnapDataHdr *)(*ppData))->size = size + 1;
((SSnapDataHdr *)(*ppData))->data[size] = '\0'; ((SSnapDataHdr *)(*ppData))->data[size] = '\0';
if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) { if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(*ppData); taosMemoryFree(*ppData);
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto _err; TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
@ -299,20 +302,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open reader if not // open reader if not
if (pReader->pMetaReader == NULL) { if (pReader->pMetaReader == NULL) {
code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader); code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = metaSnapRead(pReader->pMetaReader, ppData); code = metaSnapRead(pReader->pMetaReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err;
if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->metaDone = 1;
goto _exit; code = metaSnapReaderClose(&pReader->pMetaReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader);
if (code) goto _err;
}
} }
} }
@ -322,20 +323,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbReader == NULL) { if (pReader->pTsdbReader == NULL) {
code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges, code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges,
&pReader->pTsdbReader); &pReader->pTsdbReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapRead(pReader->pTsdbReader, ppData); code = tsdbSnapRead(pReader->pTsdbReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tsdbDone = 1;
goto _exit; code = tsdbSnapReaderClose(&pReader->pTsdbReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tsdbDone = 1;
code = tsdbSnapReaderClose(&pReader->pTsdbReader);
if (code) goto _err;
}
} }
} }
@ -344,20 +342,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbRAWReader == NULL) { if (pReader->pTsdbRAWReader == NULL) {
ASSERT(pReader->sver == 0); ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData); code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tsdbRAWDone = 1;
goto _exit; code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tsdbRAWDone = 1;
code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
if (code) goto _err;
}
} }
} }
@ -365,59 +360,53 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
vInfo("vgId:%d tq transform start", vgId); vInfo("vgId:%d tq transform start", vgId);
if (!pReader->tqHandleDone) { if (!pReader->tqHandleDone) {
if (pReader->pTqSnapReader == NULL) { if (pReader->pTqSnapReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE, &pReader->pTqSnapReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE,
if (code < 0) goto _err; &pReader->pTqSnapReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapRead(pReader->pTqSnapReader, ppData); code = tqSnapRead(pReader->pTqSnapReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tqHandleDone = 1;
goto _exit; code = tqSnapReaderClose(&pReader->pTqSnapReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tqHandleDone = 1;
code = tqSnapReaderClose(&pReader->pTqSnapReader);
if (code) goto _err;
}
} }
} }
if (!pReader->tqCheckInfoDone) { if (!pReader->tqCheckInfoDone) {
if (pReader->pTqCheckInfoReader == NULL) { if (pReader->pTqCheckInfoReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO, &pReader->pTqCheckInfoReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO,
if (code < 0) goto _err; &pReader->pTqCheckInfoReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapRead(pReader->pTqCheckInfoReader, ppData); code = tqSnapRead(pReader->pTqCheckInfoReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tqCheckInfoDone = 1;
goto _exit; code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tqCheckInfoDone = 1;
code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
if (code) goto _err;
}
} }
} }
if (!pReader->tqOffsetDone) { if (!pReader->tqOffsetDone) {
if (pReader->pTqOffsetReader == NULL) { if (pReader->pTqOffsetReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET, &pReader->pTqOffsetReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET,
if (code < 0) goto _err; &pReader->pTqOffsetReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapRead(pReader->pTqOffsetReader, ppData); code = tqSnapRead(pReader->pTqOffsetReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tqOffsetDone = 1;
goto _exit; code = tqSnapReaderClose(&pReader->pTqOffsetReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tqOffsetDone = 1;
code = tqSnapReaderClose(&pReader->pTqOffsetReader);
if (code) goto _err;
}
} }
} }
@ -426,28 +415,19 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (!pReader->streamTaskDone) { if (!pReader->streamTaskDone) {
if (pReader->pStreamTaskReader == NULL) { if (pReader->pStreamTaskReader == NULL) {
code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader); code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
vError("vgId:%d open streamtask snapshot reader failed, code:%s", vgId, tstrerror(code));
goto _err;
}
} }
code = streamTaskSnapRead(pReader->pStreamTaskReader, ppData); code = streamTaskSnapRead(pReader->pStreamTaskReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
vError("vgId:%d error happens during read data from streatask snapshot, code:%s", vgId, tstrerror(code)); if (*ppData) {
goto _err; vInfo("vgId:%d no streamTask snapshot", vgId);
goto _exit;
} else { } else {
if (*ppData) { pReader->streamTaskDone = 1;
vInfo("vgId:%d no streamTask snapshot", vgId); code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} else { pReader->pStreamTaskReader = NULL;
pReader->streamTaskDone = 1;
code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
if (code) {
goto _err;
}
pReader->pStreamTaskReader = NULL;
}
} }
} }
if (!pReader->streamStateDone) { if (!pReader->streamStateDone) {
@ -457,21 +437,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (code) { if (code) {
pReader->streamStateDone = 1; pReader->streamStateDone = 1;
pReader->pStreamStateReader = NULL; pReader->pStreamStateReader = NULL;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = streamStateSnapRead(pReader->pStreamStateReader, ppData); code = streamStateSnapRead(pReader->pStreamStateReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->streamStateDone = 1;
goto _exit; code = streamStateSnapReaderClose(pReader->pStreamStateReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->streamStateDone = 1; pReader->pStreamStateReader = NULL;
code = streamStateSnapReaderClose(pReader->pStreamStateReader);
if (code) goto _err;
pReader->pStreamStateReader = NULL;
}
} }
} }
@ -480,20 +457,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open if not // open if not
if (pReader->pRsmaReader == NULL) { if (pReader->pRsmaReader == NULL) {
code = rsmaSnapReaderOpen(pReader->pVnode->pSma, pReader->sver, pReader->ever, &pReader->pRsmaReader); code = rsmaSnapReaderOpen(pReader->pVnode->pSma, pReader->sver, pReader->ever, &pReader->pRsmaReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = rsmaSnapRead(pReader->pRsmaReader, ppData); code = rsmaSnapRead(pReader->pRsmaReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->rsmaDone = 1;
goto _exit; code = rsmaSnapReaderClose(&pReader->pRsmaReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->rsmaDone = 1;
code = rsmaSnapReaderClose(&pReader->pRsmaReader);
if (code) goto _err;
}
} }
} }
@ -501,21 +475,21 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
*nData = 0; *nData = 0;
_exit: _exit:
if (*ppData) { if (code) {
SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData); vError("vgId:%d, vnode snapshot read failed at %s:%d since %s", vgId, __FILE__, lino, tstrerror(code));
pReader->index++;
*nData = sizeof(SSnapDataHdr) + pHdr->size;
pHdr->index = pReader->index;
vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", vgId, pReader->index,
pHdr->type, *nData);
} else { } else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index); if (*ppData) {
} SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData);
return code;
_err: pReader->index++;
vError("vgId:%d, vnode snapshot read failed since %s", vgId, tstrerror(code)); *nData = sizeof(SSnapDataHdr) + pHdr->size;
pHdr->index = pReader->index;
vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", vgId, pReader->index,
pHdr->type, *nData);
} else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index);
}
}
return code; return code;
} }
@ -532,19 +506,19 @@ struct SVSnapWriter {
SMetaSnapWriter *pMetaSnapWriter; SMetaSnapWriter *pMetaSnapWriter;
// tsdb // tsdb
TFileSetRangeArray *pRanges; TFileSetRangeArray *pRanges;
STsdbSnapWriter *pTsdbSnapWriter; STsdbSnapWriter *pTsdbSnapWriter;
// tsdb raw // tsdb raw
STsdbSnapRAWWriter *pTsdbSnapRAWWriter; STsdbSnapRAWWriter *pTsdbSnapRAWWriter;
// tq // tq
STqSnapWriter *pTqSnapHandleWriter; STqSnapWriter *pTqSnapHandleWriter;
STqSnapWriter *pTqSnapOffsetWriter; STqSnapWriter *pTqSnapOffsetWriter;
STqSnapWriter *pTqSnapCheckInfoWriter; STqSnapWriter *pTqSnapCheckInfoWriter;
// stream // stream
SStreamTaskWriter *pStreamTaskWriter; SStreamTaskWriter *pStreamTaskWriter;
SStreamStateWriter *pStreamStateWriter; SStreamStateWriter *pStreamStateWriter;
// rsma // rsma
TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
SRSmaSnapWriter *pRsmaSnapWriter; SRSmaSnapWriter *pRsmaSnapWriter;
}; };
TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
@ -563,18 +537,18 @@ TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t
static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
int32_t code = -1; int32_t code = 0;
int32_t lino;
if (pParam->data) { if (pParam->data) {
SSyncTLV *datHead = (void *)pParam->data; SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT; TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
goto _out;
} }
STsdbRepOpts tsdbOpts = {0}; STsdbRepOpts tsdbOpts = {0};
TFileSetRangeArray **ppRanges = NULL; TFileSetRangeArray **ppRanges = NULL;
int32_t offset = 0; int32_t offset = 0;
while (offset + sizeof(SSyncTLV) < datHead->len) { while (offset + sizeof(SSyncTLV) < datHead->len) {
SSyncTLV *subField = (void *)(datHead->val + offset); SSyncTLV *subField = (void *)(datHead->val + offset);
@ -589,30 +563,30 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP
ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
if (ppRanges == NULL) { if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
goto _out; TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out;
} }
code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); TSDB_CHECK_CODE(code, lino, _exit);
goto _out;
}
} break; } break;
default: default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
goto _out; TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
goto _exit;
} }
} }
vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
} }
code = 0; _exit:
_out: if (code) {
vError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
}
return code; return code;
} }
@ -634,6 +608,7 @@ static int32_t vnodeEnableBgTask(SVnode *pVnode) {
int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVSnapWriter *pWriter = NULL; SVSnapWriter *pWriter = NULL;
int64_t sver = pParam->start; int64_t sver = pParam->start;
int64_t ever = pParam->end; int64_t ever = pParam->end;
@ -644,8 +619,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
// alloc // alloc
pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter)); pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) { if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
pWriter->pVnode = pVnode; pWriter->pVnode = pVnode;
pWriter->sver = sver; pWriter->sver = sver;
@ -655,19 +629,19 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
pWriter->commitID = ++pVnode->state.commitID; pWriter->commitID = ++pVnode->state.commitID;
// snapshot info // snapshot info
if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) { code = vnodeSnapWriterDealWithSnapInfo(pWriter, pParam);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
} else {
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
} }
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
return code;
_err:
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
return code; return code;
} }
@ -785,14 +759,13 @@ _exit:
static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
// decode info // decode info
if (vnodeDecodeInfo(pHdr->data, &pWriter->info) < 0) { code = vnodeDecodeInfo(pHdr->data, &pWriter->info);
code = TSDB_CODE_INVALID_MSG; TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
// change some value // change some value
pWriter->info.state.commitID = pWriter->commitID; pWriter->info.state.commitID = pWriter->commitID;
@ -805,10 +778,8 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_
pWriter->info.config = pVnode->config; pWriter->info.config = pVnode->config;
pWriter->info.config.vndStats = vndStats; pWriter->info.config.vndStats = vndStats;
vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId); vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId);
if (vnodeSaveInfo(dir, &pWriter->info) < 0) { code = vnodeSaveInfo(dir, &pWriter->info);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
_exit: _exit:
return code; return code;
@ -816,6 +787,7 @@ _exit:
int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
@ -824,7 +796,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pHdr->index != pWriter->index + 1) { if (pHdr->index != pWriter->index + 1) {
vError("vgId:%d, unexpected vnode snapshot msg. index:%" PRId64 ", expected index:%" PRId64, TD_VID(pVnode), vError("vgId:%d, unexpected vnode snapshot msg. index:%" PRId64 ", expected index:%" PRId64, TD_VID(pVnode),
pHdr->index, pWriter->index + 1); pHdr->index, pWriter->index + 1);
return -1; TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_MSG, lino, _exit);
} }
pWriter->index = pHdr->index; pWriter->index = pHdr->index;
@ -835,17 +807,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
switch (pHdr->type) { switch (pHdr->type) {
case SNAP_DATA_CFG: { case SNAP_DATA_CFG: {
code = vnodeSnapWriteInfo(pWriter, pData, nData); code = vnodeSnapWriteInfo(pWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_META: { case SNAP_DATA_META: {
// meta // meta
if (pWriter->pMetaSnapWriter == NULL) { if (pWriter->pMetaSnapWriter == NULL) {
code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData); code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TSDB: case SNAP_DATA_TSDB:
case SNAP_DATA_DEL: { case SNAP_DATA_DEL: {
@ -853,69 +825,69 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pTsdbSnapWriter == NULL) { if (pWriter->pTsdbSnapWriter == NULL) {
code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges, code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges,
&pWriter->pTsdbSnapWriter); &pWriter->pTsdbSnapWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr); code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
// tsdb // tsdb
if (pWriter->pTsdbSnapRAWWriter == NULL) { if (pWriter->pTsdbSnapRAWWriter == NULL) {
ASSERT(pWriter->sver == 0); ASSERT(pWriter->sver == 0);
code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter); code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr); code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TQ_HANDLE: { case SNAP_DATA_TQ_HANDLE: {
// tq handle // tq handle
if (pWriter->pTqSnapHandleWriter == NULL) { if (pWriter->pTqSnapHandleWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData); code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TQ_CHECKINFO: { case SNAP_DATA_TQ_CHECKINFO: {
// tq checkinfo // tq checkinfo
if (pWriter->pTqSnapCheckInfoWriter == NULL) { if (pWriter->pTqSnapCheckInfoWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData); code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TQ_OFFSET: { case SNAP_DATA_TQ_OFFSET: {
// tq offset // tq offset
if (pWriter->pTqSnapOffsetWriter == NULL) { if (pWriter->pTqSnapOffsetWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData); code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_STREAM_TASK: case SNAP_DATA_STREAM_TASK:
case SNAP_DATA_STREAM_TASK_CHECKPOINT: { case SNAP_DATA_STREAM_TASK_CHECKPOINT: {
if (pWriter->pStreamTaskWriter == NULL) { if (pWriter->pStreamTaskWriter == NULL) {
code = streamTaskSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamTaskWriter); code = streamTaskSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamTaskWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData); code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_STREAM_STATE_BACKEND: { case SNAP_DATA_STREAM_STATE_BACKEND: {
if (pWriter->pStreamStateWriter == NULL) { if (pWriter->pStreamStateWriter == NULL) {
code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter); code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = streamStateSnapWrite(pWriter->pStreamStateWriter, pData, nData); code = streamStateSnapWrite(pWriter->pStreamStateWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_RSMA1: case SNAP_DATA_RSMA1:
@ -925,20 +897,19 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pRsmaSnapWriter == NULL) { if (pWriter->pRsmaSnapWriter == NULL) {
code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges, code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges,
&pWriter->pRsmaSnapWriter); &pWriter->pRsmaSnapWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData); code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
default: default:
break; break;
} }
_exit: _exit:
return code; if (code) {
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode),
_err: tstrerror(code), pHdr->index, pHdr->type, nData);
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), }
tstrerror(code), pHdr->index, pHdr->type, nData);
return code; return code;
} }

34
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -387,7 +387,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (msgcb == NULL || msgcb->putToQueueFp == NULL) { if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return -1; return TSDB_CODE_INVALID_PARA;
} }
int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg); int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
@ -400,13 +400,13 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (pMsg == NULL || pMsg->pCont == NULL) { if (pMsg == NULL || pMsg->pCont == NULL) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (msgcb == NULL || msgcb->putToQueueFp == NULL) { if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return -1; return TSDB_CODE_INVALID_PARA;
} }
int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
@ -485,8 +485,7 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta
static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader); return vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
return code;
} }
static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) { static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
@ -496,8 +495,7 @@ static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len); return vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
return code;
} }
static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) { static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
@ -514,8 +512,7 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
} }
} while (true); } while (true);
int32_t code = vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter); return vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
return code;
} }
static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
@ -580,7 +577,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
if (pMeta->startInfo.startAllTasks == 1) { if (pMeta->startInfo.startAllTasks == 1) {
pMeta->startInfo.restartCount += 1; pMeta->startInfo.restartCount += 1;
vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
pMeta->startInfo.restartCount); pMeta->startInfo.restartCount);
} else { } else {
pMeta->startInfo.startAllTasks = 1; pMeta->startInfo.startAllTasks = 1;
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -636,7 +633,7 @@ static void vnodeBecomeLeader(const SSyncFSM *pFsm) {
} }
} }
static void vnodeBecomeAssignedLeader(const SSyncFSM* pFsm) { static void vnodeBecomeAssignedLeader(const SSyncFSM *pFsm) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
vDebug("vgId:%d, become assigned leader", pVnode->config.vgId); vDebug("vgId:%d, become assigned leader", pVnode->config.vgId);
if (pVnode->pTq) { if (pVnode->pTq) {
@ -662,12 +659,16 @@ static int32_t vnodeApplyQueueItems(const SSyncFSM *pFsm) {
int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE);
return itemSize; return itemSize;
} else { } else {
return -1; return TSDB_CODE_INVALID_PARA;
} }
} }
static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
if (pFsm == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pFsm->data = pVnode; pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitMsg; pFsm->FpCommitCb = vnodeSyncCommitMsg;
pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex; pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex;
@ -724,7 +725,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
pVnode->sync = syncOpen(&syncInfo, vnodeVersion); pVnode->sync = syncOpen(&syncInfo, vnodeVersion);
if (pVnode->sync <= 0) { if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr()); vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
return -1; return terrno;
} }
return 0; return 0;
@ -732,9 +733,10 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
int32_t vnodeSyncStart(SVnode *pVnode) { int32_t vnodeSyncStart(SVnode *pVnode) {
vInfo("vgId:%d, start sync", pVnode->config.vgId); vInfo("vgId:%d, start sync", pVnode->config.vgId);
if (syncStart(pVnode->sync) < 0) { int32_t code = syncStart(pVnode->sync);
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr()); if (code) {
return -1; vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, tstrerror(code));
return code;
} }
return 0; return 0;
} }

3
source/libs/catalog/src/catalog.c
View File

@ -446,7 +446,8 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
} }
char* pJson = parseTagDatatoJson(pTag); char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
STagVal tagVal; STagVal tagVal;
tagVal.cid = 0; tagVal.cid = 0;
tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.type = TSDB_DATA_TYPE_JSON;

3
source/libs/catalog/src/ctgAsync.c
View File

@ -2079,7 +2079,8 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf*
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
} }
char* pJson = parseTagDatatoJson(pTag); char* pJson = NULL;
parseTagDatatoJson(pTag, &pJson);
STagVal tagVal; STagVal tagVal;
tagVal.cid = 0; tagVal.cid = 0;
tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.type = TSDB_DATA_TYPE_JSON;

9
source/libs/command/inc/commandInt.h
View File

@ -176,7 +176,14 @@ typedef struct SExplainCtx {
#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
#define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column")) #define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column"))
#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) #define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p, _r) \
do { \
if ((_u) == 'n' || (_u) == 'y') { \
_r = (_t); \
} else { \
code = convertTimeFromPrecisionToUnit(_t, _p, _u, &_r); \
} \
} while(0)
#define EXPLAIN_ROW_NEW(level, ...) \ #define EXPLAIN_ROW_NEW(level, ...) \
do { \ do { \

185
source/libs/command/src/command.c
View File

@ -23,6 +23,15 @@
#include "tglobal.h" #include "tglobal.h"
#include "tgrant.h" #include "tgrant.h"
#define COL_DATA_SET_VAL_AND_CHECK(pCol, rows, buf, isNull) \
do { \
int _code = colDataSetVal(pCol, rows, buf, isNull);\
if (TSDB_CODE_SUCCESS != _code) { \
terrno = _code; \
return _code; \
} \
} while(0)
extern SConfig* tsCfg; extern SConfig* tsCfg;
static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) {
@ -100,7 +109,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock; *pOutput = pBlock;
} else { } else {
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
} }
return code; return code;
} }
@ -108,7 +117,7 @@ static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta, static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta,
int8_t biMode) { int8_t biMode) {
int32_t blockCap = (biMode != 0) ? numOfRows + 1 : numOfRows; int32_t blockCap = (biMode != 0) ? numOfRows + 1 : numOfRows;
blockDataEnsureCapacity(pBlock, blockCap); QRY_ERR_RET(blockDataEnsureCapacity(pBlock, blockCap));
pBlock->info.rows = 0; pBlock->info.rows = 0;
// field // field
@ -138,11 +147,12 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
continue; continue;
} }
STR_TO_VARSTR(buf, pMeta->schema[i].name); STR_TO_VARSTR(buf, pMeta->schema[i].name);
colDataSetVal(pCol1, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol1, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name); STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
colDataSetVal(pCol2, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol2, pBlock->info.rows, buf, false);
int32_t bytes = getSchemaBytes(pMeta->schema + i); int32_t bytes = getSchemaBytes(pMeta->schema + i);
colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false); COL_DATA_SET_VAL_AND_CHECK(pCol3, pBlock->info.rows, (const char*)&bytes, false);
if (TSDB_VIEW_TABLE != pMeta->tableType) { if (TSDB_VIEW_TABLE != pMeta->tableType) {
if (i >= pMeta->tableInfo.numOfColumns) { if (i >= pMeta->tableInfo.numOfColumns) {
STR_TO_VARSTR(buf, "TAG"); STR_TO_VARSTR(buf, "TAG");
@ -155,22 +165,22 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
} else { } else {
STR_TO_VARSTR(buf, "VIEW COL"); STR_TO_VARSTR(buf, "VIEW COL");
} }
colDataSetVal(pCol4, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false);
if (useCompress(pMeta->tableType) && pMeta->schemaExt) { if (useCompress(pMeta->tableType) && pMeta->schemaExt) {
if (i < pMeta->tableInfo.numOfColumns) { if (i < pMeta->tableInfo.numOfColumns) {
STR_TO_VARSTR(buf, columnEncodeStr(COMPRESS_L1_TYPE_U32(pMeta->schemaExt[i].compress))); STR_TO_VARSTR(buf, columnEncodeStr(COMPRESS_L1_TYPE_U32(pMeta->schemaExt[i].compress)));
colDataSetVal(pCol5, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, columnCompressStr(COMPRESS_L2_TYPE_U32(pMeta->schemaExt[i].compress))); STR_TO_VARSTR(buf, columnCompressStr(COMPRESS_L2_TYPE_U32(pMeta->schemaExt[i].compress)));
colDataSetVal(pCol6, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol6, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pMeta->schemaExt[i].compress))); STR_TO_VARSTR(buf, columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pMeta->schemaExt[i].compress)));
colDataSetVal(pCol7, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol7, pBlock->info.rows, buf, false);
} else { } else {
STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled"); STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled");
colDataSetVal(pCol5, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol5, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled"); STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled");
colDataSetVal(pCol6, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol6, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled"); STR_TO_VARSTR(buf, fillTagCol == 0 ? "" : "disabled");
colDataSetVal(pCol7, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol7, pBlock->info.rows, buf, false);
} }
} }
@ -180,13 +190,13 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
} }
if (pMeta->tableType == TSDB_SUPER_TABLE && biMode != 0) { if (pMeta->tableType == TSDB_SUPER_TABLE && biMode != 0) {
STR_TO_VARSTR(buf, "tbname"); STR_TO_VARSTR(buf, "tbname");
colDataSetVal(pCol1, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol1, pBlock->info.rows, buf, false);
STR_TO_VARSTR(buf, "VARCHAR"); STR_TO_VARSTR(buf, "VARCHAR");
colDataSetVal(pCol2, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol2, pBlock->info.rows, buf, false);
int32_t bytes = TSDB_TABLE_NAME_LEN - 1; int32_t bytes = TSDB_TABLE_NAME_LEN - 1;
colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false); COL_DATA_SET_VAL_AND_CHECK(pCol3, pBlock->info.rows, (const char*)&bytes, false);
STR_TO_VARSTR(buf, "TAG"); STR_TO_VARSTR(buf, "TAG");
colDataSetVal(pCol4, pBlock->info.rows, buf, false); COL_DATA_SET_VAL_AND_CHECK(pCol4, pBlock->info.rows, buf, false);
++(pBlock->info.rows); ++(pBlock->info.rows);
} }
if (pBlock->info.rows <= 0) { if (pBlock->info.rows <= 0) {
@ -212,7 +222,7 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp**
code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp);
} }
} }
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
return code; return code;
} }
@ -234,7 +244,7 @@ static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock; *pOutput = pBlock;
} else { } else {
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
} }
return code; return code;
} }
@ -264,13 +274,17 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
return v; return v;
} }
static char* buildRetension(SArray* pRetension) { static int32_t buildRetension(SArray* pRetension, char **ppRetentions ) {
size_t size = taosArrayGetSize(pRetension); size_t size = taosArrayGetSize(pRetension);
if (size == 0) { if (size == 0) {
return NULL; *ppRetentions = NULL;
return TSDB_CODE_SUCCESS;
} }
char* p1 = taosMemoryCalloc(1, 100); char* p1 = taosMemoryCalloc(1, 100);
if(NULL == p1) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t len = 0; int32_t len = 0;
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
@ -288,7 +302,8 @@ static char* buildRetension(SArray* pRetension) {
} }
} }
return p1; *ppRetentions = p1;
return TSDB_CODE_SUCCESS;
} }
static const char* cacheModelStr(int8_t cacheModel) { static const char* cacheModelStr(int8_t cacheModel) {
@ -319,14 +334,14 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) {
return TSDB_CACHE_MODEL_NONE_STR; return TSDB_CACHE_MODEL_NONE_STR;
} }
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
blockDataEnsureCapacity(pBlock, 1); QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
pBlock->info.rows = 1; pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0}; char buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0};
STR_TO_VARSTR(buf1, dbName); STR_TO_VARSTR(buf1, dbName);
colDataSetVal(pCol1, 0, buf1, false); COL_DATA_SET_VAL_AND_CHECK(pCol1, 0, buf1, false);
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0}; char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0};
@ -347,7 +362,8 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
break; break;
} }
char* retentions = buildRetension(pCfg->pRetensions); char* pRetentions = NULL;
QRY_ERR_RET(buildRetension(pCfg->pRetensions, &pRetentions));
int32_t dbFNameLen = strlen(dbFName); int32_t dbFNameLen = strlen(dbFName);
int32_t hashPrefix = 0; int32_t hashPrefix = 0;
if (pCfg->hashPrefix > 0) { if (pCfg->hashPrefix > 0) {
@ -374,28 +390,30 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize,
pCfg->s3KeepLocal, pCfg->s3Compact); pCfg->s3KeepLocal, pCfg->s3Compact);
if (retentions) { if (pRetentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", pRetentions);
} }
} }
taosMemoryFree(retentions); taosMemoryFree(pRetentions);
(varDataLen(buf2)) = len; (varDataLen(buf2)) = len;
colDataSetVal(pCol2, 0, buf2, false); COL_DATA_SET_VAL_AND_CHECK(pCol2, 0, buf2, false);
return TSDB_CODE_SUCCESS;
} }
static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL; SSDataBlock* pBlock = NULL;
int32_t code = buildCreateDBResultDataBlock(&pBlock); int32_t code = buildCreateDBResultDataBlock(&pBlock);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->dbFName, pStmt->pCfg); code = setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->dbFName, pStmt->pCfg);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp); code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp);
} }
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
return code; return code;
} }
@ -415,7 +433,7 @@ static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock; *pOutput = pBlock;
} else { } else {
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
} }
return code; return code;
} }
@ -436,7 +454,7 @@ static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock; *pOutput = pBlock;
} else { } else {
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
} }
return code; return code;
} }
@ -494,6 +512,7 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
} }
int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
int32_t code = TSDB_CODE_SUCCESS;
SArray* pTagVals = NULL; SArray* pTagVals = NULL;
STag* pTag = (STag*)pCfg->pTags; STag* pTag = (STag*)pCfg->pTags;
@ -503,20 +522,15 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
} }
if (tTagIsJson(pTag)) { if (tTagIsJson(pTag)) {
char* pJson = parseTagDatatoJson(pTag); char* pJson = NULL;
if (pJson) { parseTagDatatoJson(pTag, &pJson);
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
taosMemoryFree(pJson); taosMemoryFree(pJson);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = tTagToValArray((const STag*)pCfg->pTags, &pTagVals); QRY_ERR_RET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals));
if (code) {
return code;
}
int16_t valueNum = taosArrayGetSize(pTagVals); int16_t valueNum = taosArrayGetSize(pTagVals);
int32_t num = 0; int32_t num = 0;
int32_t j = 0; int32_t j = 0;
@ -534,16 +548,18 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j);
if (pSchema->colId > pTagVal->cid) { if (pSchema->colId > pTagVal->cid) {
qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid); qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid);
taosArrayDestroy(pTagVals); code = TSDB_CODE_APP_ERROR;
return TSDB_CODE_APP_ERROR; TAOS_CHECK_ERRNO(code);
} else if (pSchema->colId == pTagVal->cid) { } else if (pSchema->colId == pTagVal->cid) {
char type = pTagVal->type; char type = pTagVal->type;
int32_t tlen = 0; int32_t tlen = 0;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen);
TAOS_CHECK_ERRNO(code);
} else { } else {
dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen);
TAOS_CHECK_ERRNO(code);
} }
*len += tlen; *len += tlen;
j++; j++;
@ -580,9 +596,10 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
*/ */
} }
_exit:
taosArrayDestroy(pTagVals); taosArrayDestroy(pTagVals);
return TSDB_CODE_SUCCESS; return code;
} }
void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) {
@ -647,20 +664,19 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg*
} }
static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg) { static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* pDbCfg, char* tbName, STableCfg* pCfg) {
int32_t code = 0; int32_t code = TSDB_CODE_SUCCESS;
blockDataEnsureCapacity(pBlock, 1); QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
pBlock->info.rows = 1; pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_TB_RESULT_FIELD1_LEN] = {0}; char buf1[SHOW_CREATE_TB_RESULT_FIELD1_LEN] = {0};
STR_TO_VARSTR(buf1, tbName); STR_TO_VARSTR(buf1, tbName);
colDataSetVal(pCol1, 0, buf1, false); QRY_ERR_RET(colDataSetVal(pCol1, 0, buf1, false));
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN); char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN);
if (NULL == buf2) { if (NULL == buf2) {
terrno = TSDB_CODE_OUT_OF_MEMORY; QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
return terrno;
} }
int32_t len = 0; int32_t len = 0;
@ -677,10 +693,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
appendTagNameFields(buf2, &len, pCfg); appendTagNameFields(buf2, &len, pCfg);
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS (");
code = appendTagValues(buf2, &len, pCfg); code = appendTagValues(buf2, &len, pCfg);
if (code) { TAOS_CHECK_ERRNO(code);
taosMemoryFree(buf2);
return code;
}
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
appendTableOptions(buf2, &len, pDbCfg, pCfg); appendTableOptions(buf2, &len, pDbCfg, pCfg);
} else { } else {
@ -692,29 +705,30 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
varDataLen(buf2) = (len > 65535) ? 65535 : len; varDataLen(buf2) = (len > 65535) ? 65535 : len;
colDataSetVal(pCol2, 0, buf2, false); code = colDataSetVal(pCol2, 0, buf2, false);
TAOS_CHECK_ERRNO(code);
_exit:
taosMemoryFree(buf2); taosMemoryFree(buf2);
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreateViewStmt* pStmt) { static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreateViewStmt* pStmt) {
int32_t code = 0; int32_t code = 0;
blockDataEnsureCapacity(pBlock, 1); QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
pBlock->info.rows = 1; pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_VIEW_RESULT_FIELD1_LEN + 1] = {0}; char buf1[SHOW_CREATE_VIEW_RESULT_FIELD1_LEN + 1] = {0};
snprintf(varDataVal(buf1), TSDB_VIEW_FNAME_LEN + 4, "`%s`.`%s`", pStmt->dbName, pStmt->viewName); snprintf(varDataVal(buf1), TSDB_VIEW_FNAME_LEN + 4, "`%s`.`%s`", pStmt->dbName, pStmt->viewName);
varDataSetLen(buf1, strlen(varDataVal(buf1))); varDataSetLen(buf1, strlen(varDataVal(buf1)));
colDataSetVal(pCol1, 0, buf1, false); QRY_ERR_RET(colDataSetVal(pCol1, 0, buf1, false));
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char* buf2 = taosMemoryMalloc(SHOW_CREATE_VIEW_RESULT_FIELD2_LEN); char* buf2 = taosMemoryMalloc(SHOW_CREATE_VIEW_RESULT_FIELD2_LEN);
if (NULL == buf2) { if (NULL == buf2) {
terrno = TSDB_CODE_OUT_OF_MEMORY; QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
return terrno;
} }
SViewMeta* pMeta = pStmt->pViewMeta; SViewMeta* pMeta = pStmt->pViewMeta;
@ -723,11 +737,10 @@ static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreate
pStmt->dbName, pStmt->viewName, pMeta->querySql); pStmt->dbName, pStmt->viewName, pMeta->querySql);
int32_t len = strlen(varDataVal(buf2)); int32_t len = strlen(varDataVal(buf2));
varDataLen(buf2) = (len > 65535) ? 65535 : len; varDataLen(buf2) = (len > 65535) ? 65535 : len;
colDataSetVal(pCol2, 0, buf2, false); code = colDataSetVal(pCol2, 0, buf2, false);
taosMemoryFree(buf2); taosMemoryFree(buf2);
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
@ -739,7 +752,7 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp);
} }
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
return code; return code;
} }
@ -832,23 +845,39 @@ static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) {
pBlock->info.hasVarCol = true; pBlock->info.hasVarCol = true;
pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData));
if (NULL == pBlock->pDataBlock) {
taosMemoryFree(pBlock);
return TSDB_CODE_OUT_OF_MEMORY;
}
SColumnInfoData infoData = {0}; SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData); if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) {
goto _exit;
}
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData); if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) {
goto _exit;
}
infoData.info.type = TSDB_DATA_TYPE_VARCHAR; infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN; infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN;
taosArrayPush(pBlock->pDataBlock, &infoData); if(taosArrayPush(pBlock->pDataBlock, &infoData) == NULL) {
goto _exit;
}
*pOutput = pBlock; *pOutput = pBlock;
return TSDB_CODE_SUCCESS;
_exit:
if(terrno != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBlock);
taosArrayDestroy(pBlock->pDataBlock);
}
return terrno;
} }
static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
@ -860,7 +889,7 @@ static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp); code = buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp);
} }
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
return code; return code;
} }
@ -881,14 +910,14 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p
infoData.info.type = pExpr->resType.type; infoData.info.type = pExpr->resType.type;
infoData.info.bytes = pExpr->resType.bytes; infoData.info.bytes = pExpr->resType.bytes;
} }
blockDataAppendColInfo(pBlock, &infoData); QRY_ERR_RET(blockDataAppendColInfo(pBlock, &infoData));
} }
*pOutput = pBlock; *pOutput = pBlock;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) { int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
blockDataEnsureCapacity(pBlock, 1); QRY_ERR_RET(blockDataEnsureCapacity(pBlock, 1));
int32_t index = 0; int32_t index = 0;
SNode* pProj = NULL; SNode* pProj = NULL;
@ -897,9 +926,9 @@ int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {
return TSDB_CODE_PAR_INVALID_SELECTED_EXPR; return TSDB_CODE_PAR_INVALID_SELECTED_EXPR;
} else { } else {
if (((SValueNode*)pProj)->isNull) { if (((SValueNode*)pProj)->isNull) {
colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true); QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, NULL, true));
} else { } else {
colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false); QRY_ERR_RET(colDataSetVal(taosArrayGet(pBlock->pDataBlock, index++), 0, nodesGetValueFromNode((SValueNode*)pProj), false));
} }
} }
} }
@ -917,7 +946,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp); code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp);
} }
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
return code; return code;
} }
@ -930,7 +959,7 @@ static int32_t execShowCreateView(SShowCreateViewStmt* pStmt, SRetrieveTableRsp*
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_VIEW_RESULT_COLS, pRsp); code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_VIEW_RESULT_COLS, pRsp);
} }
blockDataDestroy(pBlock); (void)blockDataDestroy(pBlock);
return code; return code;
} }

149
source/libs/command/src/explain.c
View File

@ -55,7 +55,7 @@ char* qExplainGetAsofOpStr(int32_t opType) {
case OP_TYPE_LOWER_EQUAL: case OP_TYPE_LOWER_EQUAL:
return "<="; return "<=";
case OP_TYPE_EQUAL: case OP_TYPE_EQUAL:
return "="; return "=";
default: default:
return "UNKNOWN"; return "UNKNOWN";
} }
@ -192,14 +192,14 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
if (0 == group->physiPlanExecIdx) { if (0 == group->physiPlanExecIdx) {
group->nodeIdx = 0; group->nodeIdx = 0;
} }
rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++);
if (group->physiPlanExecIdx >= rsp->numOfPlans) { if (group->physiPlanExecIdx >= rsp->numOfPlans) {
qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno;
} else { } else {
for (int32_t i = 0; i < group->nodeNum; ++i) { for (int32_t i = 0; i < group->nodeNum; ++i) {
rsp = taosArrayGet(group->nodeExecInfo, i); rsp = taosArrayGet(group->nodeExecInfo, i);
@ -208,7 +208,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno;
} }
} }
@ -338,7 +338,7 @@ static char* qExplainGetScanMode(STableScanPhysiNode* pScan) {
isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname; isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname;
if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) {
return "seq_grp_order"; return "seq_grp_order";
} }
if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) {
return "grp_order"; return "grp_order";
@ -416,7 +416,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -581,7 +581,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -621,7 +621,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False");
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pPrjNode->node.pConditions) { if (pPrjNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
@ -629,7 +629,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -689,7 +689,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin); EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
@ -705,7 +705,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) { if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) {
@ -733,7 +733,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
if (NULL != pJoinNode->pColOnCond) { if (NULL != pJoinNode->pColOnCond) {
@ -741,7 +741,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
@ -831,7 +831,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -844,7 +844,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId);
QRY_ERR_RET(TSDB_CODE_APP_ERROR); QRY_ERR_RET(TSDB_CODE_APP_ERROR);
} }
nodeNum += group->nodeNum; nodeNum += group->nodeNum;
} }
@ -876,7 +876,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) { for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) {
@ -956,7 +956,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -991,10 +991,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1043,10 +1050,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1123,7 +1137,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1163,7 +1177,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1205,7 +1219,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1247,7 +1261,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1340,7 +1354,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1381,7 +1395,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1422,7 +1436,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1464,7 +1478,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1535,7 +1549,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1563,10 +1577,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1577,7 +1598,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1644,7 +1665,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1669,7 +1690,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE, QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
@ -1722,7 +1743,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
if (pJoinNode->pPrimKeyCond) { if (pJoinNode->pPrimKeyCond) {
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pColEqCond) { if (pJoinNode->pColEqCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1730,7 +1751,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pTagEqCond) { if (pJoinNode->pTagEqCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1738,7 +1759,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pFullOnCond) { if (pJoinNode->pFullOnCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1746,14 +1767,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
} }
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->timeRangeTarget) { if (pJoinNode->timeRangeTarget) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
@ -1920,6 +1941,7 @@ _return:
} }
int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
int32_t code = 0;
SExplainCtx *pCtx = (SExplainCtx *)ctx; SExplainCtx *pCtx = (SExplainCtx *)ctx;
int32_t rowNum = taosArrayGetSize(pCtx->rows); int32_t rowNum = taosArrayGetSize(pCtx->rows);
if (rowNum <= 0) { if (rowNum <= 0) {
@ -1929,14 +1951,14 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SSDataBlock *pBlock = createDataBlock(); SSDataBlock *pBlock = createDataBlock();
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
blockDataAppendColInfo(pBlock, &infoData); QRY_ERR_JRET(blockDataAppendColInfo(pBlock, &infoData));
blockDataEnsureCapacity(pBlock, rowNum); QRY_ERR_JRET(blockDataEnsureCapacity(pBlock, rowNum));
SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0); SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
for (int32_t i = 0; i < rowNum; ++i) { for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
colDataSetVal(pInfoData, i, row->buf, false); QRY_ERR_JRET(colDataSetVal(pInfoData, i, row->buf, false));
} }
pBlock->info.rows = rowNum; pBlock->info.rows = rowNum;
@ -1946,8 +1968,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) { if (NULL == rsp) {
qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
blockDataDestroy(pBlock); QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
rsp->completed = 1; rsp->completed = 1;
@ -1961,10 +1982,11 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
SET_PAYLOAD_LEN(rsp->data, len, len); SET_PAYLOAD_LEN(rsp->data, len, len);
_return:
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
*pRsp = rsp; *pRsp = rsp;
return TSDB_CODE_SUCCESS; QRY_RET(code);
} }
int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) {
@ -2099,33 +2121,31 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t
group->nodeExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainRsp)); group->nodeExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainRsp));
if (NULL == group->nodeExecInfo) { if (NULL == group->nodeExecInfo) {
qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); qError("taosArrayInit %d explainExecInfo failed", group->nodeNum);
tFreeSExplainRsp(pRspMsg); code = TSDB_CODE_OUT_OF_MEMORY;
taosWUnLockLatch(&group->lock); TAOS_CHECK_ERRNO(code);
QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
group->physiPlanExecNum = pRspMsg->numOfPlans; group->physiPlanExecNum = pRspMsg->numOfPlans;
} else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) { } else if (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum) {
qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo), qError("group execInfo already full, size:%d, nodeNum:%d", (int32_t)taosArrayGetSize(group->nodeExecInfo),
group->nodeNum); group->nodeNum);
tFreeSExplainRsp(pRspMsg); code = TSDB_CODE_APP_ERROR;
taosWUnLockLatch(&group->lock); TAOS_CHECK_ERRNO(code);
QRY_ERR_RET(TSDB_CODE_APP_ERROR);
} }
if (group->physiPlanExecNum != pRspMsg->numOfPlans) { if (group->physiPlanExecNum != pRspMsg->numOfPlans) {
qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum, qError("physiPlanExecNum %d mismatch with others %d in group %d", pRspMsg->numOfPlans, group->physiPlanExecNum,
groupId); groupId);
tFreeSExplainRsp(pRspMsg); code = TSDB_CODE_APP_ERROR;
taosWUnLockLatch(&group->lock); TAOS_CHECK_ERRNO(code);
}
QRY_ERR_RET(TSDB_CODE_APP_ERROR);
if(taosArrayPush(group->nodeExecInfo, pRspMsg) == NULL)
{
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_ERRNO(code);
} }
taosArrayPush(group->nodeExecInfo, pRspMsg);
groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum);
taosWUnLockLatch(&group->lock); taosWUnLockLatch(&group->lock);
@ -2140,6 +2160,11 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_exit:
tFreeSExplainRsp(pRspMsg);
taosWUnLockLatch(&group->lock);
return code;
} }
int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {

2
source/libs/executor/inc/executil.h
View File

@ -179,7 +179,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SFunctionStateStore* pStore);
void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn);
void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); int32_t initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
SColumn extractColumnFromColumnNode(SColumnNode* pColNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode);

21
source/libs/executor/inc/executorInt.h
View File

@ -834,9 +834,8 @@ void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbI
/** /**
* @brief copydata from hash table, instead of copying from SGroupResInfo's pRow * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow
*/ */
int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, void doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup);
bool ignoreGroup);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo);
@ -862,10 +861,11 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache); int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); int32_t setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId,
void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId); const char* name);
void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer); int32_t setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId);
int32_t setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer);
int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput,
int32_t* rowEntryInfoOffset); int32_t* rowEntryInfoOffset);
@ -878,7 +878,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList); int32_t numOfOutput, SArray* pPseudoList);
void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); int32_t setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
int32_t checkForQueryBuf(size_t numOfTables); int32_t checkForQueryBuf(size_t numOfTables);
@ -897,8 +898,8 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup);
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,
SStateStore* pStore); SStateStore* pStore);
void appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, int32_t appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp,
void* pTbName); void* pTbName);
uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId);

11
source/libs/executor/inc/mergejoin.h
View File

@ -432,6 +432,15 @@ typedef struct SMJoinOperatorInfo {
} \ } \
} while (0) } while (0)
#define MJ_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin); void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin);
@ -459,7 +468,7 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable); int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable); int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable);
int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp); int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp);
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build); int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont);
int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond); int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond);
int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx); int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx);
int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp); int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp);

4
source/libs/executor/inc/tfill.h
View File

@ -131,11 +131,11 @@ SFillInfo* taosCreateFillInfo(TSKEY skey, int32_t numOfFillCols, int32_t numOfNo
int32_t order, const char* id); int32_t order, const char* id);
void* taosDestroyFillInfo(struct SFillInfo* pFillInfo); void* taosDestroyFillInfo(struct SFillInfo* pFillInfo);
int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity); void taosFillResultDataBlock(struct SFillInfo* pFillInfo, SSDataBlock* p, int32_t capacity);
int64_t getFillInfoStart(struct SFillInfo* pFillInfo); int64_t getFillInfoStart(struct SFillInfo* pFillInfo);
bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData, bool fillIfWindowPseudoColumn(SFillInfo* pFillInfo, SFillColInfo* pCol, SColumnInfoData* pDstColInfoData,
int32_t rowIndex); int32_t rowIndex);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

Some files were not shown because too many files have changed in this diff Show More