Merge branch '3.0' into opti/TD-31017

This commit is contained in:
WANG MINGMING 2024-07-23 19:44:06 +08:00 committed by GitHub
commit 535755266d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
140 changed files with 8419 additions and 5257 deletions

View File

@ -49,7 +49,7 @@ typedef struct SBlockOrderInfo {
#define colDataSetNull_f_s(c_, r_) \ #define colDataSetNull_f_s(c_, r_) \
do { \ do { \
colDataSetNull_f((c_)->nullbitmap, r_); \ colDataSetNull_f((c_)->nullbitmap, r_); \
memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \ (void)memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \
} while (0) } while (0)
#define colDataClearNull_f(bm_, r_) \ #define colDataClearNull_f(bm_, r_) \

View File

@ -86,11 +86,11 @@ void deltaToUtcInitOnce();
char getPrecisionUnit(int32_t precision); char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit); int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
struct STm { struct STm {
struct tm tm; struct tm tm;

View File

@ -34,9 +34,10 @@ typedef struct {
* @brief Start one Qnode in Dnode. * @brief Start one Qnode in Dnode.
* *
* @param pOption Option of the qnode. * @param pOption Option of the qnode.
* @return SQnode* The qnode object. * @param pQnode The qnode object.
* @return int32_t The error code.
*/ */
SQnode *qndOpen(const SQnodeOpt *pOption); int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode);
/** /**
* @brief Stop Qnode in Dnode. * @brief Stop Qnode in Dnode.

View File

@ -36,7 +36,7 @@ typedef struct SFuncExecEnv {
} SFuncExecEnv; } SFuncExecEnv;
typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); typedef int32_t (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

View File

@ -138,6 +138,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_CACHE_LAST_ROW, FUNCTION_TYPE_CACHE_LAST_ROW,
FUNCTION_TYPE_CACHE_LAST, FUNCTION_TYPE_CACHE_LAST,
FUNCTION_TYPE_TABLE_COUNT, FUNCTION_TYPE_TABLE_COUNT,
FUNCTION_TYPE_GROUP_CONST_VALUE,
// distributed splitting functions // distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
@ -256,9 +257,10 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId); bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId); bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId); bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes); void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList); int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc);
@ -271,7 +273,7 @@ typedef enum EFuncDataRequired {
} EFuncDataRequired; } EFuncDataRequired;
EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo); int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus);
int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);

View File

@ -77,7 +77,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf);
// high level APIs // high level APIs
bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);

View File

@ -686,7 +686,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
// checkpoint related // checkpoint related
int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId); void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId);
int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId); int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId);
int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId); int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId);
bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId); bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId);
@ -770,9 +770,9 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask); int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer // timer
tmr_h streamTimerGetInstance(); int32_t streamTimerGetInstance(tmr_h* pTmr);
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg); const char* pMsg);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
@ -809,6 +809,9 @@ void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -61,6 +61,7 @@ typedef enum { M2C = 0, C2M } ConvType;
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) #define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src))
char *tstrdup(const char *src); char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4); int32_t taosUcs4len(TdUcs4 *ucs4);

View File

@ -860,6 +860,11 @@ int32_t taosGetErrSize();
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)
#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809) #define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809)
#define TSDB_CODE_FUNC_TIME_UNIT_INVALID TAOS_DEF_ERROR_CODE(0, 0x280A)
#define TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x280B)
#define TSDB_CODE_FUNC_INVALID_VALUE_RANGE TAOS_DEF_ERROR_CODE(0, 0x280C)
#define TSDB_CODE_FUNC_SETUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x280D)
//udf //udf
#define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901)

View File

@ -35,6 +35,7 @@
#include "tsched.h" #include "tsched.h"
#include "ttime.h" #include "ttime.h"
#include "tversion.h" #include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h" #include "cus_name.h"
@ -875,6 +876,12 @@ void taos_init_imp(void) {
} }
rpcInit(); rpcInit();
if (InitRegexCache() != 0) {
tscInitRes = -1;
tscError("failed to init regex cache");
return;
}
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
catalogInit(&cfg); catalogInit(&cfg);
@ -890,7 +897,11 @@ void taos_init_imp(void) {
tscError("failed to init task queue"); tscError("failed to init task queue");
return; return;
} }
fmFuncMgtInit(); if (fmFuncMgtInit() != TSDB_CODE_SUCCESS) {
tscInitRes = -1;
tscError("failed to init function manager");
return;
}
nodesInitAllocatorSet(); nodesInitAllocatorSet();
clientConnRefPool = taosOpenRef(200, destroyTscObj); clientConnRefPool = taosOpenRef(200, destroyTscObj);

View File

@ -28,6 +28,7 @@
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -78,6 +79,7 @@ void taos_cleanup(void) {
clientConnRefPool = -1; clientConnRefPool = -1;
taosCloseRef(id); taosCloseRef(id);
DestroyRegexCache();
rpcCleanup(); rpcCleanup();
tscDebug("rpc cleanup"); tscDebug("rpc cleanup");

View File

@ -2491,7 +2491,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
int32_t code = buildChildTableName(&rname); int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
taosArrayDestroy(tags); taosArrayDestroy(tags);

View File

@ -150,7 +150,6 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
cap -= nwrite; cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) { for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) { if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else { } else {

View File

@ -35,7 +35,7 @@ void deltaToUtcInitOnce() {
// printf("====delta:%lld\n\n", seconds); // printf("====delta:%lld\n\n", seconds);
} }
static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction);
static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
@ -95,7 +95,9 @@ char* forwardToTimeStringEnd(char* str) {
return &str[i]; return &str[i];
} }
int64_t parseFraction(char* str, char** end, int32_t timePrec) { int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t i = 0; int32_t i = 0;
int64_t fraction = 0; int64_t fraction = 0;
@ -112,7 +114,7 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
int32_t totalLen = i; int32_t totalLen = i;
if (totalLen <= 0) { if (totalLen <= 0) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
/* parse the fraction */ /* parse the fraction */
@ -134,21 +136,24 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
} }
times = NANO_SEC_FRACTION_LEN - i; times = NANO_SEC_FRACTION_LEN - i;
} else { } else {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
fraction = strnatoi(str, i) * factor[times]; fraction = strnatoi(str, i) * factor[times];
*end = str + totalLen; *end = str + totalLen;
*pFraction = fraction;
return fraction; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t parseTimezone(char* str, int64_t* tzOffset) { int32_t parseTimezone(char* str, int64_t* tzOffset) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t hour = 0; int64_t hour = 0;
int32_t i = 0; int32_t i = 0;
if (str[i] != '+' && str[i] != '-') { if (str[i] != '+' && str[i] != '-') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
i++; i++;
@ -160,7 +165,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
continue; continue;
} }
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
char* sep = strchr(&str[i], ':'); char* sep = strchr(&str[i], ':');
@ -175,18 +180,18 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
} }
if (hour > 12 || hour < 0) { if (hour > 12 || hour < 0) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
// return error if there're illegal charaters after min(2 Digits) // return error if there're illegal charaters after min(2 Digits)
char* minStr = &str[i]; char* minStr = &str[i];
if (minStr[1] != '\0' && minStr[2] != '\0') { if (minStr[1] != '\0' && minStr[2] != '\0') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
int64_t minute = strnatoi(&str[i], 2); int64_t minute = strnatoi(&str[i], 2);
if (minute > 59 || (hour == 12 && minute > 0)) { if (minute > 59 || (hour == 12 && minute > 0)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
if (str[0] == '+') { if (str[0] == '+') {
@ -195,13 +200,13 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
*tzOffset = hour * 3600 + minute * 60; *tzOffset = hour * 3600 + minute * 60;
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) { if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) {
*offset = 0; *offset = 0;
return 0; return TSDB_CODE_SUCCESS;
} }
return parseTimezone(tzStr, offset); return parseTimezone(tzStr, offset);
} }
@ -219,6 +224,8 @@ int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
* 2013-04-12T15:52:01.123+0800 * 2013-04-12T15:52:01.123+0800
*/ */
int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t factor = TSDB_TICK_PER_SECOND(timePrec);
int64_t tzOffset = 0; int64_t tzOffset = 0;
@ -234,7 +241,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
} }
if (str == NULL) { if (str == NULL) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
/* mktime will be affected by TZ, set by using taos_options */ /* mktime will be affected by TZ, set by using taos_options */
@ -253,22 +260,18 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor; *time = seconds * factor;
} else if (str[0] == '.') { } else if (str[0] == '.') {
str += 1; str += 1;
if ((fraction = parseFraction(str, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str, &str, timePrec, &fraction));
return -1;
}
*time = seconds * factor + fraction; *time = seconds * factor + fraction;
char seg = str[0]; char seg = str[0];
if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') { if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') { } else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if (seg == '+' || seg == '-') { } else if (seg == '+' || seg == '-') {
// parse the timezone // parse the timezone
if (parseTimezone(str, &tzOffset) == -1) { TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
return -1;
}
*time += tzOffset * factor; *time += tzOffset * factor;
} }
@ -277,16 +280,14 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor + fraction; *time = seconds * factor + fraction;
// parse the timezone // parse the timezone
if (parseTimezone(str, &tzOffset) == -1) { TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
return -1;
}
*time += tzOffset * factor; *time += tzOffset * factor;
} else { } else {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static FORCE_INLINE bool validateTm(struct tm* pTm) { static FORCE_INLINE bool validateTm(struct tm* pTm) {
@ -314,6 +315,8 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) {
} }
int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0; *utime = 0;
struct tm tm = {0}; struct tm tm = {0};
@ -330,7 +333,7 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
// if parse failed, try "%Y-%m-%d" format // if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm); str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
@ -347,16 +350,16 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
if (*str == '.') { if (*str == '.') {
/* parse the second fraction part */ /* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
return -1;
}
} }
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0; *utime = 0;
struct tm tm = {0}; struct tm tm = {0};
tm.tm_isdst = -1; tm.tm_isdst = -1;
@ -374,7 +377,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
// if parse failed, try "%Y-%m-%d" format // if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm); str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
@ -384,13 +387,11 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
int64_t fraction = 0; int64_t fraction = 0;
if (*str == '.') { if (*str == '.') {
/* parse the second fraction part */ /* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
return -1;
}
} }
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
char getPrecisionUnit(int32_t precision) { char getPrecisionUnit(int32_t precision) {
@ -482,10 +483,12 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre
// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double =
// 1626006833631000064 // 1626006833631000064
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO &&
fromPrecision != TSDB_TIME_PRECISION_NANO) { fromPrecision != TSDB_TIME_PRECISION_NANO) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
@ -541,15 +544,23 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
time *= factors[fromPrecision]; time *= factors[fromPrecision];
break; break;
default: { default: {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
if (tmp >= (double)INT64_MAX) return INT64_MAX; if (tmp >= (double)INT64_MAX) {
if (tmp <= (double)INT64_MIN) return INT64_MIN; *pRes = INT64_MAX;
return time; } else if (tmp <= (double)INT64_MIN) {
*pRes = INT64_MIN;
} else {
*pRes = time;
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) { int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t charLen = varDataLen(inputData); int32_t charLen = varDataLen(inputData);
char* newColData; char* newColData;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
@ -558,7 +569,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return TSDB_CODE_INVALID_TIMESTAMP; TAOS_RETURN(TSDB_CODE_INVALID_TIMESTAMP);
} }
taosMemoryFree(newColData); taosMemoryFree(newColData);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
@ -566,50 +577,52 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData);
if (len < 0) { if (len < 0) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return TSDB_CODE_FAILED; TAOS_RETURN(TSDB_CODE_FAILED);
} }
newColData[len] = 0; newColData[len] = 0;
int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight); int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return ret; TAOS_RETURN(ret);
} }
taosMemoryFree(newColData); taosMemoryFree(newColData);
} else { } else {
return TSDB_CODE_FAILED; TAOS_RETURN(TSDB_CODE_FAILED);
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
switch (unit) { switch (unit) {
case 's': case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) { if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'm': case 'm':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'h': case 'h':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'd': case 'd':
if (val > INT64_MAX / MILLISECOND_PER_DAY) { if (val > INT64_MAX / MILLISECOND_PER_DAY) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'w': case 'w':
if (val > INT64_MAX / MILLISECOND_PER_WEEK) { if (val > INT64_MAX / MILLISECOND_PER_WEEK) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
@ -623,10 +636,10 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
(*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision); (*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision);
break; break;
default: { default: {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
/* /*
@ -645,47 +658,50 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
*/ */
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit,
int32_t timePrecision) { int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0; errno = 0;
char* endPtr = NULL; char* endPtr = NULL;
/* get the basic numeric value */ /* get the basic numeric value */
int64_t timestamp = taosStr2Int64(token, &endPtr, 10); int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
if ((timestamp == 0 && token[0] != '0') || errno != 0) { if ((timestamp == 0 && token[0] != '0') || errno != 0) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
/* natual month/year are not allowed in absolute duration */ /* natual month/year are not allowed in absolute duration */
*unit = token[tokenlen - 1]; *unit = token[tokenlen - 1];
if (*unit == 'n' || *unit == 'y') { if (*unit == 'n' || *unit == 'y') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
return getDuration(timestamp, *unit, duration, timePrecision); return getDuration(timestamp, *unit, duration, timePrecision);
} }
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) { int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision,
bool negativeAllow) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0; errno = 0;
/* get the basic numeric value */ /* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10); *duration = taosStr2Int64(token, NULL, 10);
if ((*duration < 0 && !negativeAllow) || errno != 0) { if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
*unit = token[tokenLen - 1]; *unit = token[tokenLen - 1];
if (*unit == 'n' || *unit == 'y') { if (*unit == 'n' || *unit == 'y') {
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
if(isdigit(*unit)) { if (isdigit(*unit)) {
*unit = getPrecisionUnit(timePrecision); *unit = getPrecisionUnit(timePrecision);
} }
return getDuration(*duration, *unit, duration, timePrecision); return getDuration(*duration, *unit, duration, timePrecision);
} }
static bool taosIsLeapYear(int32_t year) { static bool taosIsLeapYear(int32_t year) { return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); }
return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0));
}
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
if (duration == 0) { if (duration == 0) {
@ -900,7 +916,8 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
// used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point // used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point
int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) {
if (pInterval->offset > 0) { if (pInterval->offset > 0) {
int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t slideStart =
taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision);
int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision);
return result; return result;
@ -960,7 +977,9 @@ const char* fmtts(int64_t ts) {
return buf; return buf;
} }
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
int32_t code = TSDB_CODE_SUCCESS;
char ts[40] = {0}; char ts[40] = {0};
struct tm ptm; struct tm ptm;
@ -996,17 +1015,18 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
default: default:
fractionLen = 0; fractionLen = 0;
return; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
if (taosLocalTime(&quot, &ptm, buf) == NULL) { if (taosLocalTime(&quot, &ptm, buf) == NULL) {
return; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
length += snprintf(ts + length, fractionLen, format, mod); length += snprintf(ts + length, fractionLen, format, mod);
length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
tstrncpy(buf, ts, bufLen); tstrncpy(buf, ts, bufLen);
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) {
@ -1055,7 +1075,7 @@ typedef enum {
TSFKW_Mon, TSFKW_Mon,
TSFKW_MS, TSFKW_MS,
TSFKW_NS, TSFKW_NS,
//TSFKW_OF, // TSFKW_OF,
TSFKW_PM, TSFKW_PM,
TSFKW_P_M, TSFKW_P_M,
TSFKW_SS, TSFKW_SS,
@ -1076,7 +1096,7 @@ typedef enum {
TSFKW_day, TSFKW_day,
TSFKW_ddd, TSFKW_ddd,
TSFKW_dd, TSFKW_dd,
TSFKW_dy, // mon, tue TSFKW_dy, // mon, tue
TSFKW_d, TSFKW_d,
TSFKW_hh24, TSFKW_hh24,
TSFKW_hh12, TSFKW_hh12,
@ -1268,7 +1288,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
} }
if (*formatStr == '\\' && *(formatStr + 1)) { if (*formatStr == '\\' && *(formatStr + 1)) {
formatStr++; formatStr++;
last = NULL; // stop expanding last format, create new format last = NULL; // stop expanding last format, create new format
} }
if (last) { if (last) {
// expand // expand
@ -1289,7 +1309,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
// for other strings // for other strings
if (*formatStr == '\\' && *(formatStr + 1)) { if (*formatStr == '\\' && *(formatStr + 1)) {
formatStr++; formatStr++;
lastOtherFormat = NULL; // stop expanding lastOtherFormat = NULL; // stop expanding
} else { } else {
if (lastOtherFormat && !isSeperatorChar(*formatStr)) { if (lastOtherFormat && !isSeperatorChar(*formatStr)) {
// expanding // expanding
@ -1304,8 +1324,8 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
formatStr++; formatStr++;
} else { } else {
TSFormatNode format = { TSFormatNode format = {
.type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR,
.key = NULL}; .key = NULL};
format.c = formatStr; format.c = formatStr;
format.len = 1; format.len = 1;
taosArrayPush(formats, &format); taosArrayPush(formats, &format);
@ -1318,7 +1338,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
} }
static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
int32_t size = taosArrayGetSize(formats); int32_t size = taosArrayGetSize(formats);
const char* start = s; const char* start = s;
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
TSFormatNode* format = taosArrayGet(formats, i); TSFormatNode* format = taosArrayGet(formats, i);
@ -1923,8 +1943,8 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr
return tm2char(*formats, &tm, out, outLen); return tm2char(*formats, &tm, out, outLen);
} }
int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision,
int32_t errMsgLen) { char* errMsg, int32_t errMsgLen) {
const char* sErrPos; const char* sErrPos;
int32_t fErrIdx; int32_t fErrIdx;
if (!*formats) { if (!*formats) {
@ -1977,26 +1997,27 @@ static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1,
#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] #define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97]
static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ static int64_t UNIT_MATRIX[10][11] = {/* ns, us, ms, s, min, h, d, w, month, y*/
/*ns*/ { 1, 1000, 0}, /*ns*/ {1, 1000, 0},
/*us*/ {1000, 1, 1000, 0}, /*us*/ {1000, 1, 1000, 0},
/*ms*/ { 0, 1000, 1, 1000, 0}, /*ms*/ {0, 1000, 1, 1000, 0},
/*s*/ { 0, 0, 1000, 1, 60, 0}, /*s*/ {0, 0, 1000, 1, 60, 0},
/*min*/ { 0, 0, 0, 60, 1, 60, 0}, /*min*/ {0, 0, 0, 60, 1, 60, 0},
/*h*/ { 0, 0, 0, 0, 60, 1, 1, 0}, /*h*/ {0, 0, 0, 0, 60, 1, 1, 0},
/*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, /*d*/ {0, 0, 0, 0, 0, 24, 1, 7, 1, 0},
/*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, /*w*/ {0, 0, 0, 0, 0, 0, 7, 1, -1, 0},
/*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, /*mon*/ {0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0},
/*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; /*y*/ {0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}};
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) { if (baseIdx == idx) {
if (interval < baseInterval) return false; if (interval < baseInterval) return false;
if (checkEq && interval == baseInterval) return false; if (checkEq && interval == baseInterval) return false;
return interval % baseInterval == 0; return interval % baseInterval == 0;
} }
int8_t next = baseIdx + 1; int8_t next = baseIdx + 1;
int64_t val = UNIT_MATRIX[baseIdx][next]; int64_t val = UNIT_MATRIX[baseIdx][next];
while (val != 0 && next <= idx) { while (val != 0 && next <= idx) {
if (val == -1) { if (val == -1) {
@ -2006,7 +2027,7 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
} }
if (val % baseInterval == 0 || baseInterval % val == 0) { if (val % baseInterval == 0 || baseInterval % val == 0) {
int8_t extra = baseInterval >= val ? 0 : 1; int8_t extra = baseInterval >= val ? 0 : 1;
bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1); bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1);
if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) { if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) {
next++; next++;
val = UNIT_MATRIX[baseIdx][next]; val = UNIT_MATRIX[baseIdx][next];
@ -2021,7 +2042,8 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
return false; return false;
} }
static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) { if (baseIdx == idx) {
@ -2030,7 +2052,7 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
return interval % baseInterval == 0; return interval % baseInterval == 0;
} }
int8_t next = baseIdx - 1; int8_t next = baseIdx - 1;
int64_t val = UNIT_MATRIX[baseIdx][next]; int64_t val = UNIT_MATRIX[baseIdx][next];
while (val != 0 && next >= 0) { while (val != 0 && next >= 0) {
return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq); return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq);
@ -2041,18 +2063,27 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
/* /*
* @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
* @param baseInterval, baseUnit, interval/unit of base tsma * @param baseInterval, baseUnit, interval/unit of base tsma
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already
* translated to TICKS of [precision] * been translated to TICKS of [precision]
* @param unit the tsma unit going to create * @param unit the tsma unit going to create
* @param precision the precision of this db * @param precision the precision of this db
* @param checkEq pass true if same interval is not acceptable, false if acceptable. * @param checkEq pass true if same interval is not acceptable, false if acceptable.
* @ret true the tsma can be created, else cannot * @ret true the tsma can be created, else cannot
* */ * */
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) { bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision,
bool checkEq) {
bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit); bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit);
if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit); if (!baseIsCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit, &baseInterval)) {
return false;
}
}
bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit); bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit);
if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit); if (!isCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(interval, precision, unit, &interval)) {
return false;
}
}
bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq; bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq;

View File

@ -34,11 +34,8 @@ static void qmClose(SQnodeMgmt *pMgmt) {
} }
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) { static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption); int32_t code = qndOpen(pOption, pQnode);
if (*pQnode == NULL) { return code;
return terrno;
}
return 0;
} }
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0; int32_t code = 0;
@ -62,7 +59,7 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (code != 0) { if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code)); dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt); qmClose(pMgmt);
return -1; return code;
} }
tmsgReportStartup("qnode-impl", "initialized"); tmsgReportStartup("qnode-impl", "initialized");

View File

@ -48,8 +48,8 @@ static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
} }
int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
qndPreprocessQueryMsg(pMgmt->pQnode, pMsg); int32_t code = qndPreprocessQueryMsg(pMgmt->pQnode, pMsg);
if (code) return code;
return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);
} }

View File

@ -96,57 +96,58 @@ typedef struct STaskChkptInfo {
int8_t dropHTask; int8_t dropHTask;
}STaskChkptInfo; }STaskChkptInfo;
int32_t mndInitStream(SMnode *pMnode); int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode);
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream);
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream); int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId); int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId);
int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt); int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt);
bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock); bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock);
int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId); int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams); int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream); int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode, int32_t acceptCode); int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg); int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg, STrans **pTrans1);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream);
int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
int32_t mndProcessStreamHb(SRpcMsg *pReq); int32_t mndProcessStreamHb(SRpcMsg *pReq);
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
int32_t extractStreamNodeList(SMnode *pMnode); int32_t extractStreamNodeList(SMnode *pMnode);
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated); int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated);
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq); int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId, int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId,
int64_t ts); int64_t ts);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter);
void destroyStreamTaskIter(SStreamTaskIter *pIter); void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); int32_t streamTaskIterGetCurrent(SStreamTaskIter *pIter, SStreamTask **pTask);
void mndInitExecInfo(); int32_t mndInitExecInfo();
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo); void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks); int32_t mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo);
void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo); void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo);
void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo); void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo);
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId); int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId);

View File

@ -12,173 +12,179 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndDef.h" #include "mndDef.h"
#include "mndConsumer.h" #include "mndConsumer.h"
#include "taoserror.h"
static void *freeStreamTasks(SArray *pTaskLevel); static void *freeStreamTasks(SArray *pTaskLevel);
int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tStartEncode(pEncoder) < 0) return -1; TAOS_CHECK_RETURN(tStartEncode(pEncoder));
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->name));
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->createTime));
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->updateTime));
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->version));
if (tEncodeI32(pEncoder, pObj->totalLevel) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->totalLevel));
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->smaId));
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->uid));
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->status));
if (tEncodeI8(pEncoder, pObj->conf.igExpired) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.igExpired));
if (tEncodeI8(pEncoder, pObj->conf.trigger) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.trigger));
if (tEncodeI8(pEncoder, pObj->conf.fillHistory) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.fillHistory));
if (tEncodeI64(pEncoder, pObj->conf.triggerParam) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.triggerParam));
if (tEncodeI64(pEncoder, pObj->conf.watermark) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.watermark));
if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->sourceDbUid));
if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetDbUid));
if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sourceDb));
if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetDb));
if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetSTbName));
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetStbUid));
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->fixedSinkVgId));
if (pObj->sql != NULL) { if (pObj->sql != NULL) {
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sql));
} else { } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
} }
if (pObj->ast != NULL) { if (pObj->ast != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->ast));
} else { } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
} }
if (pObj->physicalPlan != NULL) { if (pObj->physicalPlan != NULL) {
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->physicalPlan));
} else { } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
} }
int32_t sz = taosArrayGetSize(pObj->tasks); int32_t sz = taosArrayGetSize(pObj->tasks);
if (tEncodeI32(pEncoder, sz) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, sz));
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i); SArray *pArray = taosArrayGetP(pObj->tasks, i);
int32_t innerSz = taosArrayGetSize(pArray); int32_t innerSz = taosArrayGetSize(pArray);
if (tEncodeI32(pEncoder, innerSz) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz));
for (int32_t j = 0; j < innerSz; j++) { for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j); SStreamTask *pTask = taosArrayGetP(pArray, j);
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
pTask->ver = SSTREAM_TASK_VER; pTask->ver = SSTREAM_TASK_VER;
} }
if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; TAOS_CHECK_RETURN(tEncodeStreamTask(pEncoder, pTask));
} }
} }
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema));
// 3.0.20 ver =2 // 3.0.20 ver =2
if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointFreq));
if (tEncodeI8(pEncoder, pObj->igCheckUpdate) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->igCheckUpdate));
// 3.0.50 ver = 3 // 3.0.50 ver = 3
if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointId));
if (tEncodeI8(pEncoder, pObj->subTableWithoutMd5) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->subTableWithoutMd5));
if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1));
tEndEncode(pEncoder); tEndEncode(pEncoder);
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (tStartDecode(pDecoder) < 0) return -1; int32_t code = 0;
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(pDecoder));
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->name));
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->createTime));
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->updateTime));
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->version));
if (tDecodeI32(pDecoder, &pObj->totalLevel) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->totalLevel));
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->smaId));
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->uid));
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->status));
if (tDecodeI8(pDecoder, &pObj->conf.igExpired) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.igExpired));
if (tDecodeI8(pDecoder, &pObj->conf.trigger) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.trigger));
if (tDecodeI8(pDecoder, &pObj->conf.fillHistory) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.fillHistory));
if (tDecodeI64(pDecoder, &pObj->conf.triggerParam) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.triggerParam));
if (tDecodeI64(pDecoder, &pObj->conf.watermark) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.watermark));
if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->sourceDbUid));
if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetDbUid));
if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->sourceDb));
if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetDb));
if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetSTbName));
if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetStbUid));
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->fixedSinkVgId));
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->sql));
if (tDecodeCStrAlloc(pDecoder, &pObj->ast) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->ast));
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan));
if (pObj->tasks != NULL) { if (pObj->tasks != NULL) {
pObj->tasks = freeStreamTasks(pObj->tasks); pObj->tasks = freeStreamTasks(pObj->tasks);
} }
int32_t sz; int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) { TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &sz));
return -1;
}
if (sz != 0) { if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *)); pObj->tasks = taosArrayInit(sz, sizeof(void *));
if (pObj->tasks == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
int32_t innerSz; int32_t innerSz;
if (tDecodeI32(pDecoder, &innerSz) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &innerSz));
SArray *pArray = taosArrayInit(innerSz, sizeof(void *)); SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
for (int32_t j = 0; j < innerSz; j++) { if (pArray != NULL) {
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); for (int32_t j = 0; j < innerSz; j++) {
if (pTask == NULL) { SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
taosArrayDestroy(pArray); if (pTask == NULL) {
return -1; taosArrayDestroy(pArray);
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
if ((code = tDecodeStreamTask(pDecoder, pTask)) < 0) {
taosMemoryFree(pTask);
taosArrayDestroy(pArray);
TAOS_RETURN(code);
}
taosArrayPush(pArray, &pTask);
} }
if (tDecodeStreamTask(pDecoder, pTask) < 0) {
taosMemoryFree(pTask);
taosArrayDestroy(pArray);
return -1;
}
taosArrayPush(pArray, &pTask);
} }
taosArrayPush(pObj->tasks, &pArray); taosArrayPush(pObj->tasks, &pArray);
} }
} }
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema));
// 3.0.20 // 3.0.20
if (sver >= 2) { if (sver >= 2) {
if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointFreq));
if (!tDecodeIsEnd(pDecoder)) { if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeI8(pDecoder, &pObj->igCheckUpdate) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->igCheckUpdate));
} }
} }
if (sver >= 3) { if (sver >= 3) {
if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointId));
} }
if (sver >= 5) { if (sver >= 5) {
if (tDecodeI8(pDecoder, &pObj->subTableWithoutMd5) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->subTableWithoutMd5));
} }
if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->reserve));
tEndDecode(pDecoder); tEndDecode(pDecoder);
return 0; TAOS_RETURN(code);
} }
void *freeStreamTasks(SArray *pTaskLevel) { void *freeStreamTasks(SArray *pTaskLevel) {
@ -220,7 +226,10 @@ void tFreeStreamObj(SStreamObj *pStream) {
SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEpNew == NULL) return NULL; if (pVgEpNew == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pVgEpNew->vgId = pVgEp->vgId; pVgEpNew->vgId = pVgEp->vgId;
// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); // pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
pVgEpNew->epSet = pVgEp->epSet; pVgEpNew->epSet = pVgEp->epSet;

View File

@ -165,15 +165,24 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep); mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep);
pRaw = mndDnodeActionEncode(&dnodeObj); pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL; pRaw = NULL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
@ -563,10 +572,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
SStatisReq statisReq = {0}; SStatisReq statisReq = {0};
int32_t code = -1; int32_t code = -1;
if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq));
terrno = TSDB_CODE_INVALID_MSG;
return code;
}
if (tsMonitorLogProtocol) { if (tsMonitorLogProtocol) {
mInfo("process statis req,\n %s", statisReq.pCont); mInfo("process statis req,\n %s", statisReq.pCont);
@ -586,23 +592,28 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
int32_t code = 0; int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj");
if (pTrans == NULL) { if (pTrans == NULL) {
code = terrno; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _exit; goto _exit;
} }
pDnode->updateTime = taosGetTimestampMs(); pDnode->updateTime = taosGetTimestampMs();
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _exit;
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, tstrerror(code));
code = terrno; code = terrno;
goto _exit; goto _exit;
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code));
code = terrno;
goto _exit; goto _exit;
} }
@ -617,10 +628,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
int32_t code = -1; int32_t code = -1;
if (tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
int64_t clusterid = mndGetClusterId(pMnode); int64_t clusterid = mndGetClusterId(pMnode);
if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) { if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) {
@ -634,6 +642,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp); pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp);
if (pDnode == NULL) { if (pDnode == NULL) {
mInfo("dnode:%s, not created yet", statusReq.dnodeEp); mInfo("dnode:%s, not created yet", statusReq.dnodeEp);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
} else { } else {
@ -871,16 +881,25 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", pCreate->fqdn, pCreate->port); snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", pCreate->fqdn, pCreate->port);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep); mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
pRaw = mndDnodeActionEncode(&dnodeObj); pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL; pRaw = NULL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
@ -901,7 +920,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet));
if (NULL == rsp.dnodeList) { if (NULL == rsp.dnodeList) {
mError("failed to alloc epSet while process dnode list req"); mError("failed to alloc epSet while process dnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -922,7 +941,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp); int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -935,12 +954,12 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
mError("failed to get dnode list since %s", terrstr()); mError("failed to get dnode list since %s", tstrerror(code));
} }
tFreeSDnodeListRsp(&rsp); tFreeSDnodeListRsp(&rsp);
return code; TAOS_RETURN(code);
} }
static void getSlowLogScopeString(int32_t scope, char* result){ static void getSlowLogScopeString(int32_t scope, char* result){
@ -980,7 +999,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo)); rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo));
if (NULL == rsp.variables) { if (NULL == rsp.variables) {
mError("failed to alloc SVariablesInfo array while process show variables req"); mError("failed to alloc SVariablesInfo array while process show variables req");
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -1036,7 +1055,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp); int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -1049,11 +1068,11 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
mError("failed to get show variables info since %s", terrstr()); mError("failed to get show variables info since %s", tstrerror(code));
} }
tFreeSShowVariablesRsp(&rsp); tFreeSShowVariablesRsp(&rsp);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
@ -1062,23 +1081,17 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SCreateDnodeReq createReq = {0}; SCreateDnodeReq createReq = {0};
if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0 || (terrno = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) { if ((code = grantCheck(TSDB_GRANT_DNODE)) != 0 || (code = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) {
code = terrno;
goto _OVER; goto _OVER;
} }
if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE), NULL, _OVER);
goto _OVER;
}
if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) {
terrno = TSDB_CODE_MND_INVALID_DNODE_EP; code = TSDB_CODE_MND_INVALID_DNODE_EP;
goto _OVER; goto _OVER;
} }
@ -1086,7 +1099,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port); snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep); pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode != NULL) { if (pDnode != NULL) {
terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; code = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
@ -1103,12 +1116,12 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr()); mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, tstrerror(code));
} }
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSCreateDnodeReq(&createReq); tFreeSCreateDnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
extern int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq); extern int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq);
@ -1126,44 +1139,56 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
STrans *pTrans = NULL; STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force); mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
pRaw = mndDnodeActionEncode(pDnode); pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING);
pRaw = NULL; pRaw = NULL;
pRaw = mndDnodeActionEncode(pDnode); pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
pRaw = NULL; pRaw = NULL;
if (pMObj != NULL) { if (pMObj != NULL) {
mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force), NULL, _OVER);
} }
if (pQObj != NULL) { if (pQObj != NULL) {
mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id); mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force), NULL, _OVER);
} }
if (pSObj != NULL) { if (pSObj != NULL) {
mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id); mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force), NULL, _OVER);
} }
if (numOfVnodes > 0) { if (numOfVnodes > 0) {
mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe), NULL, _OVER);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
code = 0; code = 0;
@ -1171,7 +1196,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
return code; TAOS_RETURN(code);
} }
static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) { static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) {
@ -1209,16 +1234,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
SSnodeObj *pSObj = NULL; SSnodeObj *pSObj = NULL;
SDropDnodeReq dropReq = {0}; SDropDnodeReq dropReq = {0};
if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port, mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port,
dropReq.force ? "true" : "false", dropReq.unsafe ? "true" : "false"); dropReq.force ? "true" : "false", dropReq.unsafe ? "true" : "false");
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
goto _OVER;
}
bool force = dropReq.force; bool force = dropReq.force;
if (dropReq.unsafe) { if (dropReq.unsafe) {
@ -1232,7 +1252,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port); snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep); pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = err; code = err;
goto _OVER; goto _OVER;
} }
} }
@ -1242,11 +1262,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pMObj != NULL) { if (pMObj != NULL) {
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES; code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER; goto _OVER;
} }
if (pMnode->selfDnodeId == dropReq.dnodeId) { if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER; code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER; goto _OVER;
} }
} }
@ -1255,16 +1275,16 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs());
if (isonline && force) { if (isonline && force) {
terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; code = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER; goto _OVER;
} }
bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id); bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id);
if (!isonline && !force && !isEmpty) { if (!isonline && !force && !isEmpty) {
terrno = TSDB_CODE_DNODE_OFFLINE; code = TSDB_CODE_DNODE_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER; goto _OVER;
} }
@ -1279,7 +1299,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
} }
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
@ -1287,11 +1307,11 @@ _OVER:
mndReleaseQnode(pMnode, pQObj); mndReleaseQnode(pMnode, pQObj);
mndReleaseSnode(pMnode, pSObj); mndReleaseSnode(pMnode, pSObj);
tFreeSDropDnodeReq(&dropReq); tFreeSDropDnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
terrno = 0; int32_t code = 0;
char *p = pMCfgReq->config; char *p = pMCfgReq->config;
while (*p) { while (*p) {
if (*p == ' ') { if (*p == ' ') {
@ -1314,12 +1334,12 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
strcpy(pDCfgReq->value, pMCfgReq->value); strcpy(pDCfgReq->value, pMCfgReq->value);
} }
return 0; TAOS_RETURN(code);
_err: _err:
mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; TAOS_RETURN(code);
} }
static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) {
@ -1349,23 +1369,21 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq
} }
if (code == -1) { if (code == -1) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
} }
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0}; SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int8_t updateIpWhiteList = 0; int8_t updateIpWhiteList = 0;
mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
SDCfgDnodeReq dcfgReq = {0}; SDCfgDnodeReq dcfgReq = {0};
@ -1381,26 +1399,26 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) {
mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]",
cfgReq.dnodeId, flag); cfgReq.dnodeId, flag);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
strcpy(dcfgReq.config, "s3blocksize"); strcpy(dcfgReq.config, "s3blocksize");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
#endif #endif
} else { } else {
if (mndMCfg2DCfg(&cfgReq, &dcfgReq)) goto _err_out; TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out);
if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) {
mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
goto _err_out; goto _err_out;
} }
if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) { if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) {
updateIpWhiteList = 1; updateIpWhiteList = 1;
} }
if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out; TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), NULL, _err_out);
} }
{ // audit { // audit
@ -1412,15 +1430,15 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
int32_t code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);
// dont care suss or succ; // dont care suss or succ;
if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode); if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode);
return code; TAOS_RETURN(code);
_err_out: _err_out:
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
@ -1503,17 +1521,16 @@ _exit:
} }
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) { static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
int32_t code = 0;
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0}; SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
const STraceId *trace = &pReq->info.traceId; const STraceId *trace = &pReq->info.traceId;
SDCfgDnodeReq dcfgReq = {0}; SDCfgDnodeReq dcfgReq = {0};
@ -1523,13 +1540,13 @@ static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq); return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq);
} else { } else {
terrno = TSDB_CODE_PAR_INTERNAL_ERROR; code = TSDB_CODE_PAR_INTERNAL_ERROR;
tFreeSMCfgDnodeReq(&cfgReq); tFreeSMCfgDnodeReq(&cfgReq);
return -1; TAOS_RETURN(code);
} }
#else #else
return 0; TAOS_RETURN(code);
#endif #endif
} }
@ -1710,7 +1727,7 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) {
// get int32_t value from 'SMCfgDnodeReq' // get int32_t value from 'SMCfgDnodeReq'
static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) {
terrno = 0; int32_t code = 0;
if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) {
goto _err; goto _err;
} }
@ -1725,12 +1742,12 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32
*pOutValue = atoi(pMCfgReq->value); *pOutValue = atoi(pMCfgReq->value);
} }
return 0; TAOS_RETURN(code);
_err: _err:
mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
terrno = TSDB_CODE_INVALID_CFG; code = TSDB_CODE_INVALID_CFG;
return -1; TAOS_RETURN(code);
} }
SArray *mndGetAllDnodeFqdns(SMnode *pMnode) { SArray *mndGetAllDnodeFqdns(SMnode *pMnode) {

View File

@ -29,12 +29,14 @@ void reportStartup(const char *name, const char *desc) {}
void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR; int32_t code = 0;
return -1; code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
} }
int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR; int32_t code = 0;
return -1; code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
} }
char *i642str(int64_t val) { char *i642str(int64_t val) {

View File

@ -223,6 +223,7 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) {
} }
static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) { static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) {
terrno = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName); SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName);
if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@ -240,7 +241,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
int32_t code = -1; int32_t code = -1;
STrans *pTrans = NULL; STrans *pTrans = NULL;
if ((terrno = grantCheck(TSDB_GRANT_USER)) < 0) { if ((code = grantCheck(TSDB_GRANT_USER)) < 0) {
return code; return code;
} }
@ -260,7 +261,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
func.codeSize = pCreate->codeLen; func.codeSize = pCreate->codeLen;
func.pCode = taosMemoryMalloc(func.codeSize); func.pCode = taosMemoryMalloc(func.codeSize);
if (func.pCode == NULL || func.pCode == NULL) { if (func.pCode == NULL || func.pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
@ -270,7 +271,11 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
memcpy(func.pCode, pCreate->pCode, func.codeSize); memcpy(func.pCode, pCreate->pCode, func.codeSize);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name); mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name);
SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name); SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name);
@ -279,31 +284,61 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
func.createdTime = oldFunc->createdTime; func.createdTime = oldFunc->createdTime;
SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc); SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; if (pRedoRaw == NULL) {
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY), NULL, _OVER);
SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc); SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; if (pUndoRaw == NULL) {
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY), NULL, _OVER);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; if (pCommitRaw == NULL) {
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER);
} else { } else {
SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); SSdbRaw *pRedoRaw = mndFuncActionEncode(&func);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; if (pRedoRaw == NULL) {
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING), NULL, _OVER);
SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); SSdbRaw *pUndoRaw = mndFuncActionEncode(&func);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; if (pUndoRaw == NULL) {
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED), NULL, _OVER);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; if (pCommitRaw == NULL) {
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -315,32 +350,48 @@ _OVER:
taosMemoryFree(func.pCode); taosMemoryFree(func.pCode);
taosMemoryFree(func.pComment); taosMemoryFree(func.pComment);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name);
SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc);
if (pRedoRaw == NULL) goto _OVER; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); (void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc);
if (pUndoRaw == NULL) goto _OVER; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
(void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc);
if (pCommitRaw == NULL) goto _OVER; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -355,18 +406,14 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = NULL; SFuncObj *pFunc = NULL;
SCreateFuncReq createReq = {0}; SCreateFuncReq createReq = {0};
if (tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
#ifdef WINDOWS #ifdef WINDOWS
terrno = TSDB_CODE_MND_INVALID_PLATFORM; code = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER; goto _OVER;
#endif #endif
mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen); mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC), NULL, _OVER);
goto _OVER;
}
pFunc = mndAcquireFunc(pMnode, createReq.name); pFunc = mndAcquireFunc(pMnode, createReq.name);
if (pFunc != NULL) { if (pFunc != NULL) {
@ -378,7 +425,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
mInfo("func:%s, replace function is set", createReq.name); mInfo("func:%s, replace function is set", createReq.name);
code = 0; code = 0;
} else { } else {
terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; code = TSDB_CODE_MND_FUNC_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
} else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) {
@ -386,22 +433,22 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
} }
if (createReq.name[0] == 0) { if (createReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; code = TSDB_CODE_MND_INVALID_FUNC_NAME;
goto _OVER; goto _OVER;
} }
if (createReq.pCode == NULL) { if (createReq.pCode == NULL) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; code = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER; goto _OVER;
} }
if (createReq.codeLen <= 1) { if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; code = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER; goto _OVER;
} }
if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; code = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
goto _OVER; goto _OVER;
} }
@ -410,12 +457,12 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("func:%s, failed to create since %s", createReq.name, terrstr()); mError("func:%s, failed to create since %s", createReq.name, tstrerror(code));
} }
mndReleaseFunc(pMnode, pFunc); mndReleaseFunc(pMnode, pFunc);
tFreeSCreateFuncReq(&createReq); tFreeSCreateFuncReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
@ -424,18 +471,13 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = NULL; SFuncObj *pFunc = NULL;
SDropFuncReq dropReq = {0}; SDropFuncReq dropReq = {0};
if (tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("func:%s, start to drop", dropReq.name); mInfo("func:%s, start to drop", dropReq.name);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC), NULL, _OVER);
goto _OVER;
}
if (dropReq.name[0] == 0) { if (dropReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; code = TSDB_CODE_MND_INVALID_FUNC_NAME;
goto _OVER; goto _OVER;
} }
@ -446,7 +488,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
code = 0; code = 0;
goto _OVER; goto _OVER;
} else { } else {
terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; code = TSDB_CODE_MND_FUNC_NOT_EXIST;
goto _OVER; goto _OVER;
} }
} }
@ -456,11 +498,11 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("func:%s, failed to drop since %s", dropReq.name, terrstr()); mError("func:%s, failed to drop since %s", dropReq.name, tstrerror(code));
} }
mndReleaseFunc(pMnode, pFunc); mndReleaseFunc(pMnode, pFunc);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
@ -470,25 +512,25 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
SRetrieveFuncRsp retrieveRsp = {0}; SRetrieveFuncRsp retrieveRsp = {0};
if (tDeserializeSRetrieveFuncReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { if (tDeserializeSRetrieveFuncReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; code = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs; retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs;
retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo)); retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo));
if (retrieveRsp.pFuncInfos == NULL) { if (retrieveRsp.pFuncInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo)); retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo));
if (retrieveRsp.pFuncExtraInfos == NULL) { if (retrieveRsp.pFuncExtraInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
@ -497,6 +539,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName);
if (pFunc == NULL) { if (pFunc == NULL) {
if (terrno != 0) code = terrno;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
@ -541,7 +584,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp); int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp);
void *pRsp = rpcMallocCont(contLen); void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER; goto RETRIEVE_FUNC_OVER;
} }
@ -556,7 +599,7 @@ RETRIEVE_FUNC_OVER:
tFreeSRetrieveFuncReq(&retrieveReq); tFreeSRetrieveFuncReq(&retrieveReq);
tFreeSRetrieveFuncRsp(&retrieveRsp); tFreeSRetrieveFuncRsp(&retrieveRsp);
return code; TAOS_RETURN(code);
} }
static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) { static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) {

View File

@ -93,6 +93,7 @@ static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName,
} }
int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -110,18 +111,20 @@ int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStb
if (pReq == NULL) { if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
STransAction action = {0}; STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pReq; action.pCont = pReq;
action.contLen = contLen; action.contLen = contLen;
action.msgType = TDMT_VND_CREATE_INDEX; action.msgType = TDMT_VND_CREATE_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
return -1; TAOS_RETURN(code);
} }
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -164,6 +167,7 @@ _err:
return NULL; return NULL;
} }
int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -182,6 +186,8 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
if (pReq == NULL) { if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1; return -1;
} }
STransAction action = {0}; STransAction action = {0};
@ -189,7 +195,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
action.pCont = pReq; action.pCont = pReq;
action.contLen = len; action.contLen = len;
action.msgType = TDMT_VND_DROP_INDEX; action.msgType = TDMT_VND_DROP_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
@ -198,7 +204,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
return 0; TAOS_RETURN(code);
} }
void mndCleanupIdx(SMnode *pMnode) { void mndCleanupIdx(SMnode *pMnode) {
@ -332,61 +338,91 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
} }
int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) { code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) {
sdbFreeRaw(pRedoRaw); sdbFreeRaw(pRedoRaw);
return -1; return -1;
} }
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { code = TSDB_CODE_MND_RETURN_VALUE_NULL;
sdbFreeRaw(pCommitRaw); if (terrno != 0) code = terrno;
return -1; TAOS_RETURN(code);
} }
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
sdbFreeRaw(pCommitRaw);
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
// static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { // static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
@ -427,10 +463,7 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL; SDbObj *pDb = NULL;
SCreateTagIndexReq createReq = {0}; SCreateTagIndexReq createReq = {0};
if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("idx:%s start to create", createReq.idxName); mInfo("idx:%s start to create", createReq.idxName);
// if (mndCheckCreateIdxReq(&createReq) != 0) { // if (mndCheckCreateIdxReq(&createReq) != 0) {
@ -439,29 +472,30 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByStb(pMnode, createReq.stbName); pDb = mndAcquireDbByStb(pMnode, createReq.stbName);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
pStb = mndAcquireStb(pMnode, createReq.stbName); pStb = mndAcquireStb(pMnode, createReq.stbName);
if (pStb == NULL) { if (pStb == NULL) {
mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName); mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName);
code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
SSIdx idx = {0}; SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx) == 0) { if ((code = mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx; pIdx = idx.pIdx;
} else { } else {
goto _OVER; goto _OVER;
} }
if (pIdx != NULL) { if (pIdx != NULL) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
goto _OVER;
}
code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb); code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb);
if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) { if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) {
@ -472,32 +506,42 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stb:%s, failed to create since %s", createReq.idxName, terrstr()); mError("stb:%s, failed to create since %s", createReq.idxName, tstrerror(code));
} }
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);
mndReleaseIdx(pMnode, pIdx); mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
return code; TAOS_RETURN(code);
} }
int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
return 0; TAOS_RETURN(code);
} }
static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) { static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) {
@ -592,6 +636,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName, static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName,
int on) { int on) {
int32_t code = 0;
taosRLockLatch(&pOld->lock); taosRLockLatch(&pOld->lock);
memcpy(pNew, pOld, sizeof(SStbObj)); memcpy(pNew, pOld, sizeof(SStbObj));
taosRUnLockLatch(&pOld->lock); taosRUnLockLatch(&pOld->lock);
@ -604,28 +649,24 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
int8_t hasIdx = 0; int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx); int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx);
if (tag < 0) { if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST; code = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
col_id_t colId = pOld->pTags[tag].colId; col_id_t colId = pOld->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId));
return -1; TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew));
}
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;
}
SSchema *pTag = pNew->pTags + tag; SSchema *pTag = pNew->pTags + tag;
if (on == 1) { if (on == 1) {
if (hasIdx && tag != 0) { if (hasIdx && tag != 0) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} else { } else {
SSCHMEA_SET_IDX_ON(pTag); SSCHMEA_SET_IDX_ON(pTag);
} }
} else { } else {
if (hasIdx == 0) { if (hasIdx == 0) {
terrno = TSDB_CODE_MND_SMA_NOT_EXIST; code = TSDB_CODE_MND_SMA_NOT_EXIST;
} else { } else {
SSCHMEA_SET_IDX_OFF(pTag); SSCHMEA_SET_IDX_OFF(pTag);
pTag->flags = 0; pTag->flags = 0;
@ -634,11 +675,15 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->tagVer++; pNew->tagVer++;
SSdbRaw *pCommitRaw = mndStbActionEncode(pNew); SSdbRaw *pCommitRaw = mndStbActionEncode(pNew);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0; TAOS_RETURN(code);
} }
int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
// impl later // impl later
@ -649,17 +694,17 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
// mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); // mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name);
mndTransSetDbName(pTrans, pDb->name, pStb->name); mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1), NULL, _OVER);
if (mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -671,7 +716,7 @@ _OVER:
taosMemoryFree(newStb.pCmpr); taosMemoryFree(newStb.pCmpr);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) { int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) {
// build index on first tag, and no index name; // build index on first tag, and no index name;
@ -725,25 +770,23 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
int8_t hasIdx = 0; int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx); int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx);
if (tag < 0) { if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST; code = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
int8_t exist = 0; int8_t exist = 0;
if (tag == 0 && hasIdx == 1) { if (tag == 0 && hasIdx == 1) {
exist = mndCheckIndexNameByTagName(pMnode, &idxObj); exist = mndCheckIndexNameByTagName(pMnode, &idxObj);
if (exist) { if (exist) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
} else if (hasIdx == 1) { } else if (hasIdx == 1) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
col_id_t colId = pStb->pTags[tag].colId; col_id_t colId = pStb->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId) != 0) { TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId));
return -1;
}
// SSchema *pTag = pStb->pTags + tag; // SSchema *pTag = pStb->pTags + tag;
// if (IS_IDX_ON(pTag)) { // if (IS_IDX_ON(pTag)) {
@ -752,7 +795,7 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
// } // }
code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj); code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj);
return code; TAOS_RETURN(code);
} }
static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) { static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) {
@ -763,22 +806,30 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
SStbObj newObj = {0}; SStbObj newObj = {0};
pStb = mndAcquireStb(pMnode, pIdx->stb); pStb = mndAcquireStb(pMnode, pIdx->stb);
if (pStb == NULL) goto _OVER; if (pStb == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name);
mndTransSetDbName(pTrans, pDb->name, NULL); mndTransSetDbName(pTrans, pDb->name, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0), NULL, _OVER);
if (mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
@ -789,7 +840,7 @@ _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);
return code; TAOS_RETURN(code);
} }
int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) { int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -798,13 +849,10 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SIdxObj *pIdx = NULL; SIdxObj *pIdx = NULL;
SDropTagIndexReq req = {0}; SDropTagIndexReq req = {0};
if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req) != 0) { TAOS_CHECK_GOTO(tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("idx:%s, start to drop", req.name); mInfo("idx:%s, start to drop", req.name);
SSIdx idx = {0}; SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx) == 0) { if ((code = mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx; pIdx = idx.pIdx;
} else { } else {
goto _OVER; goto _OVER;
@ -815,7 +863,7 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
code = 0; code = 0;
goto _OVER; goto _OVER;
} else { } else {
terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST; code = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST;
goto _OVER; goto _OVER;
} }
} }
@ -823,23 +871,22 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByIdx(pMnode, req.name); pDb = mndAcquireDbByIdx(pMnode, req.name);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
goto _OVER;
}
code = mndDropIdx(pMnode, pReq, pDb, pIdx); code = mndDropIdx(pMnode, pReq, pDb, pIdx);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("idx:%s, failed to drop since %s", req.name, terrstr()); mError("idx:%s, failed to drop since %s", req.name, tstrerror(code));
} }
mndReleaseIdx(pMnode, pIdx); mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) { static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
// do nothing // do nothing
@ -847,6 +894,7 @@ static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
} }
int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
@ -856,17 +904,17 @@ int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
if (pIter == NULL) break; if (pIter == NULL) break;
if (pIdx->stbUid == pStb->uid) { if (pIdx->stbUid == pStb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
return -1; TAOS_RETURN(code);
} }
} }
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) { int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) {
@ -891,6 +939,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb
return -1; return -1;
} }
int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
@ -900,15 +949,15 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
if (pIter == NULL) break; if (pIter == NULL) break;
if (pIdx->dbUid == pDb->uid) { if (pIdx->dbUid == pDb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; TAOS_RETURN(code);
} }
} }
sdbRelease(pSdb, pIdx); sdbRelease(pSdb, pIdx);
} }
return 0; TAOS_RETURN(code);
} }

View File

@ -30,7 +30,7 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA); SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA);
SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX); SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX);
terrno = 0; int32_t code = 0;
if (pSma == NULL && pIdx == NULL) return 0; if (pSma == NULL && pIdx == NULL) return 0;
@ -40,8 +40,8 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pSma; idx->pIdx = pSma;
} else { } else {
mndReleaseSma(pMnode, pSma); mndReleaseSma(pMnode, pSma);
terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; code = TSDB_CODE_MND_SMA_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
} else { } else {
if (type == SDB_IDX) { if (type == SDB_IDX) {
@ -49,9 +49,9 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pIdx; idx->pIdx = pIdx;
} else { } else {
mndReleaseIdx(pMnode, pIdx); mndReleaseIdx(pMnode, pIdx);
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1; TAOS_RETURN(code);
} }
} }
return 0; TAOS_RETURN(code);
} }

View File

@ -18,10 +18,11 @@
#include "systable.h" #include "systable.h"
static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) { static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema)); SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) { if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
@ -35,10 +36,11 @@ static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t co
} }
*pDst = schema; *pDst = schema;
return 0; TAOS_RETURN(code);
} }
static int32_t mndInsInitMeta(SHashObj *hash) { static int32_t mndInsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0}; STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName)); tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -55,24 +57,23 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
meta.numOfColumns = pInfosTableMeta[i].colNum; meta.numOfColumns = pInfosTableMeta[i].colNum;
meta.sysInfo = pInfosTableMeta[i].sysInfo; meta.sysInfo = pInfosTableMeta[i].sysInfo;
if (mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)) { TAOS_CHECK_RETURN(mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas));
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo, int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo,
STableMetaRsp *pRsp) { STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) { if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *pMeta = NULL; STableMetaRsp *pMeta = NULL;
@ -84,40 +85,41 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
if (!sysinfo && pMeta->sysInfo) { if (!sysinfo && pMeta->sysInfo) {
mError("no permission to get schema of table name:%s", tbName); mError("no permission to get schema of table name:%s", tbName);
terrno = TSDB_CODE_PAR_PERMISSION_DENIED; code = TSDB_CODE_PAR_PERMISSION_DENIED;
return -1; TAOS_RETURN(code);
} }
*pRsp = *pMeta; *pRsp = *pMeta;
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0; TAOS_RETURN(code);
} }
int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) { if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR; code = TSDB_CODE_APP_ERROR;
return -1; TAOS_RETURN(code);
} }
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; TAOS_RETURN(code);
} }
strcpy(pRsp->tbName, pMeta->tbName); strcpy(pRsp->tbName, pMeta->tbName);
@ -129,22 +131,21 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) { if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL; pRsp->pSchemas = NULL;
return -1; TAOS_RETURN(code);
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt)); pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt));
return 0; TAOS_RETURN(code);
} }
int32_t mndInitInfos(SMnode *pMnode) { int32_t mndInitInfos(SMnode *pMnode) {
pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pMnode->infosMeta == NULL) { if (pMnode->infosMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
return mndInsInitMeta(pMnode->infosMeta); return mndInsInitMeta(pMnode->infosMeta);

View File

@ -50,8 +50,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0; int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock); taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) { if (pMnode->stopped) {
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
code = -1;
} else if (!mndIsLeader(pMnode)) { } else if (!mndIsLeader(pMnode)) {
code = -1; code = -1;
} else { } else {
@ -63,7 +62,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
#endif #endif
} }
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
return code; TAOS_RETURN(code);
} }
static inline void mndReleaseRpc(SMnode *pMnode) { static inline void mndReleaseRpc(SMnode *pMnode) {
@ -78,6 +77,7 @@ static inline void mndReleaseRpc(SMnode *pMnode) {
} }
static void *mndBuildTimerMsg(int32_t *pContLen) { static void *mndBuildTimerMsg(int32_t *pContLen) {
terrno = 0;
SMTimerReq timerReq = {0}; SMTimerReq timerReq = {0};
int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq); int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq);
@ -329,6 +329,7 @@ static int32_t minCronTime() {
return min <= 1 ? 2 : min; return min <= 1 ? 2 : min;
} }
void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
int32_t code = 0;
if (sec % tsTtlPushIntervalSec == 0) { if (sec % tsTtlPushIntervalSec == 0) {
mndPullupTtl(pMnode); mndPullupTtl(pMnode);
} }
@ -378,14 +379,14 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
} }
if (sec % (tsArbHeartBeatIntervalSec) == 0) { if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) { if ((code = mndPullupArbHeartbeat(pMnode)) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr()); mError("failed to pullup arb heartbeat, since:%s", tstrerror(code));
} }
} }
if (sec % (tsArbCheckSyncIntervalSec) == 0) { if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) { if ((code = mndPullupArbCheckSync(pMnode)) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr()); mError("failed to pullup arb check sync, since:%s", tstrerror(code));
} }
} }
} }
@ -426,17 +427,18 @@ static void *mndThreadFp(void *param) {
} }
static int32_t mndInitTimer(SMnode *pMnode) { static int32_t mndInitTimer(SMnode *pMnode) {
int32_t code = 0;
TdThreadAttr thAttr; TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr); taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode) != 0) { if ((code = taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode)) != 0) {
mError("failed to create timer thread since %s", strerror(errno)); mError("failed to create timer thread since %s", strerror(errno));
return -1; TAOS_RETURN(code);
} }
taosThreadAttrDestroy(&thAttr); taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("mnode-timer", "initialized"); tmsgReportStartup("mnode-timer", "initialized");
return 0; TAOS_RETURN(code);
} }
static void mndCleanupTimer(SMnode *pMnode) { static void mndCleanupTimer(SMnode *pMnode) {
@ -447,21 +449,23 @@ static void mndCleanupTimer(SMnode *pMnode) {
} }
static int32_t mndCreateDir(SMnode *pMnode, const char *path) { static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
int32_t code = 0;
pMnode->path = taosStrdup(path); pMnode->path = taosStrdup(path);
if (pMnode->path == NULL) { if (pMnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
if (taosMkDir(pMnode->path) != 0) { if (taosMkDir(pMnode->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndInitWal(SMnode *pMnode) { static int32_t mndInitWal(SMnode *pMnode) {
int32_t code = 0;
char path[PATH_MAX + 20] = {0}; char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = {.vgId = 1, SWalCfg cfg = {.vgId = 1,
@ -475,12 +479,13 @@ static int32_t mndInitWal(SMnode *pMnode) {
.encryptKey = {0}}; .encryptKey = {0}};
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL) { if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL) ? tsiEncryptAlgorithm : 0; cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if (tsEncryptKey[0] == '\0') { if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1; TAOS_RETURN(code);
} else { }
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
} }
} }
@ -488,11 +493,13 @@ static int32_t mndInitWal(SMnode *pMnode) {
pMnode->pWal = walOpen(path, &cfg); pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) { if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path); code = TSDB_CODE_MND_RETURN_VALUE_NULL;
return -1; if (terrno != 0) code = terrno;
mError("failed to open wal since %s. wal:%s", tstrerror(code), path);
TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static void mndCloseWal(SMnode *pMnode) { static void mndCloseWal(SMnode *pMnode) {
@ -503,6 +510,7 @@ static void mndCloseWal(SMnode *pMnode) {
} }
static int32_t mndInitSdb(SMnode *pMnode) { static int32_t mndInitSdb(SMnode *pMnode) {
int32_t code = 0;
SSdbOpt opt = {0}; SSdbOpt opt = {0};
opt.path = pMnode->path; opt.path = pMnode->path;
opt.pMnode = pMnode; opt.pMnode = pMnode;
@ -510,10 +518,12 @@ static int32_t mndInitSdb(SMnode *pMnode) {
pMnode->pSdb = sdbInit(&opt); pMnode->pSdb = sdbInit(&opt);
if (pMnode->pSdb == NULL) { if (pMnode->pSdb == NULL) {
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndOpenSdb(SMnode *pMnode) { static int32_t mndOpenSdb(SMnode *pMnode) {
@ -539,48 +549,47 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle
step.initFp = initFp; step.initFp = initFp;
step.cleanupFp = cleanupFp; step.cleanupFp = cleanupFp;
if (taosArrayPush(pMnode->pSteps, &step) == NULL) { if (taosArrayPush(pMnode->pSteps, &step) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
return 0; TAOS_RETURN(0);
} }
static int32_t mndInitSteps(SMnode *pMnode) { static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal));
if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb));
if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans));
if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster));
if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode));
if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode));
if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode));
if (mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup));
if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode));
if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser));
if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant));
if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege));
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct));
if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream));
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic));
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer));
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe));
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup));
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb));
if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma));
if (mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx));
if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos));
if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs));
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb));
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc));
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView));
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact));
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail));
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL));
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile));
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow));
if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery));
if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync));
if (mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem) != 0) return -1; TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem));
return 0; return 0;
} }
@ -606,17 +615,16 @@ static void mndCleanupSteps(SMnode *pMnode, int32_t pos) {
} }
static int32_t mndExecSteps(SMnode *pMnode) { static int32_t mndExecSteps(SMnode *pMnode) {
int32_t code = 0;
int32_t size = taosArrayGetSize(pMnode->pSteps); int32_t size = taosArrayGetSize(pMnode->pSteps);
for (int32_t pos = 0; pos < size; pos++) { for (int32_t pos = 0; pos < size; pos++) {
SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, pos); SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, pos);
if (pStep->initFp == NULL) continue; if (pStep->initFp == NULL) continue;
if ((*pStep->initFp)(pMnode) != 0) { if ((code = (*pStep->initFp)(pMnode)) != 0) {
int32_t code = terrno; mError("%s exec failed since %s, start to cleanup", pStep->name, tstrerror(code));
mError("%s exec failed since %s, start to cleanup", pStep->name, terrstr());
mndCleanupSteps(pMnode, pos); mndCleanupSteps(pMnode, pos);
terrno = code; TAOS_RETURN(code);
return -1;
} else { } else {
mInfo("%s is initialized", pStep->name); mInfo("%s is initialized", pStep->name);
tmsgReportStartup(pStep->name, "initialized"); tmsgReportStartup(pStep->name, "initialized");
@ -624,7 +632,7 @@ static int32_t mndExecSteps(SMnode *pMnode) {
} }
pMnode->clusterId = mndGetClusterId(pMnode); pMnode->clusterId = mndGetClusterId(pMnode);
return 0; TAOS_RETURN(0);
} }
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
@ -639,6 +647,7 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
} }
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
terrno = 0;
mInfo("start to open mnode in %s", path); mInfo("start to open mnode in %s", path);
SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode)); SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode));
@ -665,7 +674,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
int32_t code = mndCreateDir(pMnode, path); int32_t code = mndCreateDir(pMnode, path);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode); mndClose(pMnode);
terrno = code; terrno = code;
return NULL; return NULL;
@ -674,7 +683,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndInitSteps(pMnode); code = mndInitSteps(pMnode);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode); mndClose(pMnode);
terrno = code; terrno = code;
return NULL; return NULL;
@ -683,7 +692,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndExecSteps(pMnode); code = mndExecSteps(pMnode);
if (code != 0) { if (code != 0) {
code = terrno; code = terrno;
mError("failed to open mnode since %s", terrstr()); mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode); mndClose(pMnode);
terrno = code; terrno = code;
return NULL; return NULL;
@ -759,45 +768,47 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
int32_t code = syncProcessMsg(pMgmt->sync, pMsg); int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
if (code != 0) { if (code != 0) {
mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType), mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
terrstr(), code); tstrerror(code), code);
} }
return code; return code;
} }
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0; int32_t code = 0;
if (!IsReq(pMsg)) TAOS_RETURN(code);
if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY || if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY ||
pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT || pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT ||
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK || pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK ||
pMsg->msgType == TDMT_SCH_TASK_NOTIFY) { pMsg->msgType == TDMT_SCH_TASK_NOTIFY) {
return 0; TAOS_RETURN(code);
} }
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
taosThreadRwlockRdlock(&pMnode->lock); taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) { if (pMnode->stopped) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_APP_IS_STOPPING; code = TSDB_CODE_APP_IS_STOPPING;
return -1; TAOS_RETURN(code);
} }
terrno = 0; terrno = 0;
SSyncState state = syncGetState(pMnode->syncMgmt.sync); SSyncState state = syncGetState(pMnode->syncMgmt.sync);
if (terrno != 0) { if (terrno != 0) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
return -1; code = terrno;
TAOS_RETURN(code);
} }
if (state.state != TAOS_SYNC_STATE_LEADER) { if (state.state != TAOS_SYNC_STATE_LEADER) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_NOT_LEADER; code = TSDB_CODE_SYN_NOT_LEADER;
goto _OVER; goto _OVER;
} }
if (!state.restored || !pMnode->restored) { if (!state.restored || !pMnode->restored) {
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_RESTORING; code = TSDB_CODE_SYN_RESTORING;
goto _OVER; goto _OVER;
} }
@ -809,7 +820,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
#endif #endif
taosThreadRwlockUnlock(&pMnode->lock); taosThreadRwlockUnlock(&pMnode->lock);
return 0; TAOS_RETURN(code);
_OVER: _OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
@ -821,19 +832,17 @@ _OVER:
pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state)); pMnode->stopped, state.restored, syncStr(state.state));
return -1; TAOS_RETURN(code);
} }
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SEpSet epSet = {0}; SEpSet epSet = {0};
int32_t tmpCode = terrno;
mndGetMnodeEpSet(pMnode, &epSet); mndGetMnodeEpSet(pMnode, &epSet);
terrno = tmpCode;
mGDebug( mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "
"role:%s, redirect numOfEps:%d inUse:%d, type:%s", "role:%s, redirect numOfEps:%d inUse:%d, type:%s",
pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored, pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code), pMnode->restored, pMnode->stopped, state.restored,
syncStr(state.state), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType)); syncStr(state.state), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType));
if (epSet.numOfEps <= 0) return -1; if (epSet.numOfEps <= 0) return -1;
@ -850,7 +859,7 @@ _OVER:
pMsg->info.rspLen = contLen; pMsg->info.rspLen = contLen;
} }
return -1; TAOS_RETURN(code);
} }
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) { int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
@ -864,12 +873,12 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
fpExt = pMnode->msgFpExt[TMSG_INDEX(pMsg->msgType)]; fpExt = pMnode->msgFpExt[TMSG_INDEX(pMsg->msgType)];
if (fpExt == NULL) { if (fpExt == NULL) {
mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
return -1; TAOS_RETURN(code);
} }
} }
if (mndCheckMnodeState(pMsg) != 0) return -1; TAOS_CHECK_RETURN(mndCheckMnodeState(pMsg));
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
if (fp) if (fp)
@ -883,6 +892,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
} else if (code == 0) { } else if (code == 0) {
mGTrace("msg:%p, successfully processed", pMsg); mGTrace("msg:%p, successfully processed", pMsg);
} else { } else {
//TODO removve this wrong set code
if (code == -1) { if (code == -1) {
code = terrno; code = terrno;
} }
@ -890,7 +900,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
} }
return code; TAOS_RETURN(code);
} }
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
@ -922,7 +932,8 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) { SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpc(pMnode) != 0) return -1; int32_t code = 0;
TAOS_CHECK_RETURN(mndAcquireRpc(pMnode));
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs(); int64_t ms = taosGetTimestampMs();
@ -934,7 +945,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL || if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) { pStbInfo->stbs == NULL) {
mndReleaseRpc(pMnode); mndReleaseRpc(pMnode);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
// cluster info // cluster info
@ -1060,7 +1073,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
} }
mndReleaseRpc(pMnode); mndReleaseRpc(pMnode);
return 0; TAOS_RETURN(code);
} }
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {

View File

@ -31,17 +31,18 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow);
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq); static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) { int32_t mndInitShow(SMnode *pMnode) {
int32_t code = 0;
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show"); pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
if (pMgmt->cache == NULL) { if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr()); mError("failed to alloc show cache since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq); mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq);
return 0; TAOS_RETURN(code);
} }
void mndCleanupShow(SMnode *pMnode) { void mndCleanupShow(SMnode *pMnode) {
@ -212,6 +213,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
} }
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowObj *pShow = NULL; SShowObj *pShow = NULL;
@ -220,10 +222,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t rowsRead = 0; int32_t rowsRead = 0;
mDebug("mndProcessRetrieveSysTableReq start"); mDebug("mndProcessRetrieveSysTableReq start");
SRetrieveTableReq retrieveReq = {0}; SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb); mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
@ -232,17 +231,17 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (pMeta == NULL) { if (pMeta == NULL) {
pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) { if (pMeta == NULL) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
} }
pShow = mndCreateShowObj(pMnode, &retrieveReq); pShow = mndCreateShowObj(pMnode, &retrieveReq);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", terrstr()); mError("failed to process show-meta req since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
pShow->pMeta = pMeta; pShow->pMeta = pMeta;
@ -250,9 +249,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; code = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
} }
@ -264,9 +263,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
if (retrieveFp == NULL) { if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
@ -275,14 +274,16 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
} }
if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { code = -1;
return -1; if (retrieveReq.db[0] &&
(code = mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db)) != 0) {
TAOS_RETURN(code);
} }
if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) { if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) {
if(strcmp(pReq->info.conn.user, "root") != 0){ if(strcmp(pReq->info.conn.user, "root") != 0){
mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type); mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type);
terrno = TSDB_CODE_MND_NO_RIGHTS; code = TSDB_CODE_MND_NO_RIGHTS;
return -1; TAOS_RETURN(code);
} }
} }
@ -308,11 +309,11 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead); rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead);
if (rowsRead < 0) { if (rowsRead < 0) {
terrno = rowsRead; code = rowsRead;
mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true); mndReleaseShowObj(pShow, true);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; TAOS_RETURN(code);
} }
pBlock->info.rows = rowsRead; pBlock->info.rows = rowsRead;
@ -325,10 +326,10 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; TAOS_RETURN(code);
} }
pRsp->handle = htobe64(pShow->id); pRsp->handle = htobe64(pShow->id);

File diff suppressed because it is too large Load Diff

View File

@ -170,38 +170,54 @@ static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew
} }
static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0}; SDCreateSnodeReq createReq = {0};
createReq.dnodeId = pDnode->id; createReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq);
@ -212,23 +228,24 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_CREATE_SNODE; action.msgType = TDMT_DND_CREATE_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +256,12 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_SNODE; action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) { if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) {
@ -256,23 +273,27 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
snodeObj.updateTime = snodeObj.createdTime; snodeObj.updateTime = snodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeRedoLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeUndoLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeCommitLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
@ -282,32 +303,27 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateSnodeReq createReq = {0}; SMCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("snode:%d, start to create", createReq.dnodeId); mInfo("snode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE), NULL, _OVER);
goto _OVER;
}
// pObj = mndAcquireSnode(pMnode, createReq.dnodeId); // pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
// if (pObj != NULL) { // if (pObj != NULL) {
// terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; // terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
// goto _OVER; // goto _OVER;
// } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { // } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) {
// goto _OVER; // goto _OVER;
// } // }
if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){ if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){
terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; code = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
@ -316,41 +332,52 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); mError("snode:%d, failed to create since %s", createReq.dnodeId, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
// mndReleaseSnode(pMnode, pObj); // mndReleaseSnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -361,20 +388,20 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
action.msgType = TDMT_DND_DROP_SNODE; action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) { int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeRedoLogs(pTrans, pObj));
if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeCommitLogs(pTrans, pObj));
if (!force) { if (!force) {
if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj));
} }
return 0; return 0;
} }
@ -383,18 +410,22 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
@ -403,23 +434,20 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
SSnodeObj *pObj = NULL; SSnodeObj *pObj = NULL;
SMDropSnodeReq dropReq = {0}; SMDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("snode:%d, start to drop", dropReq.dnodeId); mInfo("snode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireSnode(pMnode, dropReq.dnodeId); pObj = mndAcquireSnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -429,12 +457,12 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); mError("snode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
} }
mndReleaseSnode(pMnode, pObj); mndReleaseSnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

File diff suppressed because it is too large Load Diff

View File

@ -134,17 +134,18 @@ int32_t mndInitStream(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
mndInitExecInfo(); int32_t code = mndInitExecInfo();
if (code) {
if (sdbSetTable(pMnode->pSdb, table) != 0) { return code;
return -1;
} }
if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { code = sdbSetTable(pMnode->pSdb, table);
return -1; if (code) {
return terrno;
} }
return 0; code = sdbSetTable(pMnode->pSdb, tableSeq);
return code;
} }
void mndCleanupStream(SMnode *pMnode) { void mndCleanupStream(SMnode *pMnode) {
@ -251,13 +252,15 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
return 0; return 0;
} }
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream) {
SSdb *pSdb = pMnode->pSdb; terrno = 0;
SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { SSdb *pSdb = pMnode->pSdb;
(*pStream) = sdbAcquire(pSdb, SDB_STREAM, streamName);
if ((*pStream) == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
} }
return pStream; return terrno;
} }
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
@ -530,9 +533,21 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
} }
int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create task iter for stream:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
}
if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { if (mndPersistTaskDeployReq(pTrans, pTask) < 0) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return -1;
@ -706,7 +721,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
char *sql = NULL; char *sql = NULL;
int32_t sqlLen = 0; int32_t sqlLen = 0;
const char *pMsg = "create stream tasks on dnodes"; const char *pMsg = "create stream tasks on dnodes";
int32_t code = 0;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
SCMCreateStreamReq createReq = {0}; SCMCreateStreamReq createReq = {0};
@ -726,8 +741,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
pStream = mndAcquireStream(pMnode, createReq.name); code = mndAcquireStream(pMnode, createReq.name, &pStream);
if (pStream != NULL) { if (pStream != NULL || code == 0) {
if (createReq.igExists) { if (createReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createReq.name); mInfo("stream:%s, already exist, ignore exist is set", createReq.name);
goto _OVER; goto _OVER;
@ -760,8 +775,9 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg); STrans *pTrans = NULL;
if (pTrans == NULL) { code = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg, &pTrans);
if (pTrans == NULL || code) {
goto _OVER; goto _OVER;
} }
@ -802,11 +818,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
// add into buffer firstly // add into buffer firstly
// to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name); mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name);
saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo);
// mndRegisterConsensusChkptId(execInfo.pStreamConsensus, streamObj.uid); streamMutexUnlock(&execInfo.lock);
taosThreadMutexUnlock(&execInfo.lock);
// execute creation // execute creation
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
@ -867,7 +882,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
{ // check the max checkpoint id from all vnodes. { // check the max checkpoint id from all vnodes.
int64_t maxCheckpointId = -1; int64_t maxCheckpointId = -1;
if (lock) { if (lock) {
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
} }
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
@ -888,7 +903,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
} }
if (lock) { if (lock) {
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
} }
if (maxCheckpointId > maxChkptId) { if (maxCheckpointId > maxChkptId) {
@ -989,11 +1004,13 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, STrans *pTrans = NULL;
"gen checkpoint for stream"); code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME,
if (pTrans == NULL) { "gen checkpoint for stream", &pTrans);
if (pTrans == NULL || code) {
code = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); tstrerror(code));
goto _ERR; goto _ERR;
} }
@ -1033,7 +1050,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) { if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) {
return code; goto _ERR;
} }
if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) { if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) {
@ -1057,13 +1074,13 @@ int32_t extractStreamNodeList(SMnode *pMnode) {
static bool taskNodeIsUpdated(SMnode *pMnode) { static bool taskNodeIsUpdated(SMnode *pMnode) {
// check if the node update happens or not // check if the node update happens or not
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int32_t numOfNodes = extractStreamNodeList(pMnode); int32_t numOfNodes = extractStreamNodeList(pMnode);
if (numOfNodes == 0) { if (numOfNodes == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
execInfo.ts = taosGetTimestampSec(); execInfo.ts = taosGetTimestampSec();
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return false; return false;
} }
@ -1071,17 +1088,22 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i);
if (pNodeEntry->stageUpdated) { if (pNodeEntry->stageUpdated) {
mDebug("stream task not ready due to node update detected, checkpoint not issued"); mDebug("stream task not ready due to node update detected, checkpoint not issued");
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return true; return true;
} }
} }
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
mWarn("not all vnodes ready, quit from vnodes status check"); mWarn("not all vnodes ready, quit from vnodes status check");
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return true; return true;
} }
@ -1097,7 +1119,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
mDebug("stream tasks not ready due to node update"); mDebug("stream tasks not ready due to node update");
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return nodeUpdated; return nodeUpdated;
} }
@ -1107,7 +1129,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
return -1; return -1;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
if (taosArrayGetSize(execInfo.pNodeList) == 0) { if (taosArrayGetSize(execInfo.pNodeList) == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
@ -1152,7 +1174,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
removeTasksInBuf(pInvalidList, &execInfo); removeTasksInBuf(pInvalidList, &execInfo);
taosArrayDestroy(pInvalidList); taosArrayDestroy(pInvalidList);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return ready ? 0 : -1; return ready ? 0 : -1;
} }
@ -1215,14 +1237,14 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
continue; continue;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid); int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid);
if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) { if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) {
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
continue; continue;
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration}; SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration};
taosArrayPush(pList, &in); taosArrayPush(pList, &in);
@ -1265,8 +1287,9 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
for (int32_t i = 0; i < numOfQual; ++i) { for (int32_t i = 0; i < numOfQual; ++i) {
SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i); SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i);
SStreamObj *p = mndGetStreamObj(pMnode, pCheckpointInfo->streamId); SStreamObj *p = NULL;
if (p != NULL) { code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p);
if (p != NULL && code == 0) {
code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true);
sdbRelease(pSdb, p); sdbRelease(pSdb, p);
@ -1289,6 +1312,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
SMDropStreamReq dropReq = {0}; SMDropStreamReq dropReq = {0};
if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) {
@ -1299,8 +1323,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
mDebug("recv drop stream:%s msg", dropReq.name); mDebug("recv drop stream:%s msg", dropReq.name);
pStream = mndAcquireStream(pMnode, dropReq.name); code = mndAcquireStream(pMnode, dropReq.name, &pStream);
if (pStream == NULL) { if (pStream == NULL || code != 0) {
if (dropReq.igNotExists) { if (dropReq.igNotExists) {
mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name); mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1356,15 +1380,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = NULL;
if (pTrans == NULL) { code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code) {
mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr()); mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq); tFreeMDropStreamReq(&dropReq);
return -1; return -1;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid);
// drop all tasks // drop all tasks
if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
@ -1857,9 +1882,9 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfRows = 0; int32_t numOfRows = 0;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo); mndInitStreamExecInfo(pMnode, &execInfo);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
while (numOfRows < rowsCapacity) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
@ -1876,11 +1901,24 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
} }
// add row for each task // add row for each task
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
while (streamTaskIterNextTask(pIter)) { int32_t code = createStreamTaskIter(pStream, &pIter);
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); if (code) {
taosRUnLockLatch(&pStream->lock);
sdbRelease(pSdb, pStream);
mError("failed to create task iter for stream:%s", pStream->name);
continue;
}
int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
break;
}
code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
numOfRows++; numOfRows++;
} }
@ -1906,6 +1944,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter) {
static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
SMPauseStreamReq pauseReq = {0}; SMPauseStreamReq pauseReq = {0};
if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) { if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) {
@ -1913,9 +1952,8 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
pStream = mndAcquireStream(pMnode, pauseReq.name); code = mndAcquireStream(pMnode, pauseReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (pStream == NULL) {
if (pauseReq.igNotExists) { if (pauseReq.igNotExists) {
mInfo("stream:%s, not exist, not pause stream", pauseReq.name); mInfo("stream:%s, not exist, not pause stream", pauseReq.name);
return 0; return 0;
@ -1955,7 +1993,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
{ // check for tasks, if tasks are not ready, not allowed to pause { // check for tasks, if tasks are not ready, not allowed to pause
bool found = false; bool found = false;
bool readyToPause = true; bool readyToPause = true;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
@ -1978,7 +2016,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
found = true; found = true;
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
if (!found) { if (!found) {
mError("stream:%s task not report status yet, not ready for pause", pauseReq.name); mError("stream:%s task not report status yet, not ready for pause", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1992,42 +2030,49 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
} }
} }
STrans *pTrans = STrans *pTrans = NULL;
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream"); code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream", &pTrans);
if (pTrans == NULL) { if (pTrans == NULL || code) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
// if nodeUpdate happened, not send pause trans // if nodeUpdate happened, not send pause trans
if (mndStreamSetPauseAction(pMnode, pTrans, pStream) < 0) { code = mndStreamSetPauseAction(pMnode, pTrans, pStream);
if (code) {
mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr()); mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// pause stream // pause stream
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
pStream->status = STREAM_STATUS__PAUSE; pStream->status = STREAM_STATUS__PAUSE;
if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY) < 0) { code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code) {
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2039,6 +2084,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return -1; return -1;
@ -2050,9 +2096,8 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
pStream = mndAcquireStream(pMnode, resumeReq.name); code = mndAcquireStream(pMnode, resumeReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (pStream == NULL) {
if (resumeReq.igNotExists) { if (resumeReq.igNotExists) {
mInfo("stream:%s not exist, not resume stream", resumeReq.name); mInfo("stream:%s not exist, not resume stream", resumeReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2081,22 +2126,28 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = STrans *pTrans = NULL;
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream"); code =
if (pTrans == NULL) { doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream", &pTrans);
if (pTrans == NULL || code) {
mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
// set the resume action // set the resume action
if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) { if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) {
mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// resume stream // resume stream
@ -2107,7 +2158,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
@ -2115,7 +2166,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2189,6 +2240,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
STrans *pTrans = NULL; STrans *pTrans = NULL;
int32_t code = 0;
// conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool // conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool
while (1) { while (1) {
@ -2215,12 +2267,11 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
// here create only one trans // here create only one trans
if (pTrans == NULL) { if (pTrans == NULL) {
pTrans = code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets", &pTrans);
doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets"); if (pTrans == NULL || code) {
if (pTrans == NULL) {
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return terrno; return terrno = code;
} }
mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid);
@ -2237,7 +2288,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
pStream->name, pTrans->id); pStream->name, pTrans->id);
int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans);
// todo: not continue, drop all and retry again // todo: not continue, drop all and retry again
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -2252,7 +2303,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; return code;
} }
} }
@ -2261,16 +2312,17 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0; return 0;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
@ -2287,9 +2339,21 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
SStreamTaskIter *pTaskIter = createStreamTaskIter(pStream); SStreamTaskIter *pTaskIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pTaskIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
sdbRelease(pSdb, pStream);
mError("failed to create task iter for stream:%s", pStream->name);
continue;
}
while (streamTaskIterNextTask(pTaskIter)) { while (streamTaskIterNextTask(pTaskIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pTaskIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pTaskIter, &pTask);
if (code) {
break;
}
SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
epsetAssign(&entry.epset, &pTask->info.epSet); epsetAssign(&entry.epset, &pTask->info.epSet);
@ -2336,9 +2400,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int32_t numOfNodes = extractStreamNodeList(pMnode); int32_t numOfNodes = extractStreamNodeList(pMnode);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
if (numOfNodes == 0) { if (numOfNodes == 0) {
mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing"); mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing");
@ -2348,7 +2412,13 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
} }
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
if (code) {
mError("failed to take the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
atomic_store_32(&mndNodeCheckSentinel, 0); atomic_store_32(&mndNodeCheckSentinel, 0);
@ -2356,7 +2426,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
return 0; return 0;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot);
@ -2380,7 +2450,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
} }
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(changeInfo.pUpdateNodeList); taosArrayDestroy(changeInfo.pUpdateNodeList);
taosHashCleanup(changeInfo.pDBMap); taosHashCleanup(changeInfo.pDBMap);
@ -2406,9 +2476,19 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
} }
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create task iter for stream:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
break;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
@ -2478,10 +2558,11 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId); mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream);
if (pStream == NULL || code != 0) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf", mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf",
req.streamId); req.streamId);
@ -2492,7 +2573,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return -1; return -1;
} else { } else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
@ -2537,7 +2618,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
{ {
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)}; SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)};
@ -2597,10 +2678,11 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId); req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream);
if (pStream == NULL || code != 0) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId); mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer // not in meta-store yet, try to acquire the task in exec buffer
@ -2610,7 +2692,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return -1; return -1;
} else { } else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
@ -2642,7 +2724,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS); doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS);
return 0; return 0;
@ -2707,7 +2789,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// req.nodeId, req.streamId, req.taskId, req.checkpointId); // req.nodeId, req.streamId, req.taskId, req.checkpointId);
// //
// // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
// taosThreadMutexLock(&execInfo.lock); // streamMutexLock(&execInfo.lock);
// //
// // mnode handle the create stream transaction too slow may cause this problem // // mnode handle the create stream transaction too slow may cause this problem
// SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); // SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
@ -2721,7 +2803,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// if (p == NULL) { // if (p == NULL) {
// mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId); // mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId);
// terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; // terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// //
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return -1; // return -1;
@ -2737,7 +2819,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// //
// int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream); // int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
// if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly // if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs);
// //
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
@ -2754,7 +2836,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks); // SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks);
// mndAddConsensusTasks(pInfo, &req); // mndAddConsensusTasks(pInfo, &req);
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
// } // }
@ -2764,7 +2846,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId); // req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId);
// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs);
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
// } // }
@ -2777,7 +2859,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// mndReleaseStream(pMnode, pStream); // mndReleaseStream(pMnode, pStream);
// } // }
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
//} //}
@ -2790,15 +2872,21 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
mDebug("start to process consensus-checkpointId in tmr"); mDebug("start to process consensus-checkpointId in tmr");
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process"); mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process");
taosArrayDestroy(pStreamList); taosArrayDestroy(pStreamList);
return 0; return 0;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
void *pIter = NULL; void *pIter = NULL;
while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) {
@ -2808,8 +2896,9 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
int32_t num = taosArrayGetSize(pInfo->pTaskList); int32_t num = taosArrayGetSize(pInfo->pTaskList);
SArray *pList = taosArrayInit(4, sizeof(int32_t)); SArray *pList = taosArrayInit(4, sizeof(int32_t));
SStreamObj *pStream = mndGetStreamObj(pMnode, pInfo->streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { // stream has been dropped already code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) { // stream has been dropped already
mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId); mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId);
taosArrayDestroy(pList); taosArrayDestroy(pList);
continue; continue;
@ -2868,14 +2957,14 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) {
int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i); int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i);
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId); code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(pStreamList); taosArrayDestroy(pStreamList);
mDebug("end to process consensus-checkpointId in tmr"); mDebug("end to process consensus-checkpointId in tmr");
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) { static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
@ -2926,32 +3015,41 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
} }
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) { int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, STrans *pTrans = NULL;
"update checkpoint-info"); int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME,
if (pTrans == NULL) { "update checkpoint-info", &pTrans);
return terrno; if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
} }
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); if (code){
if (code != 0) { sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
} }
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) { if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);

View File

@ -61,15 +61,23 @@ void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
} }
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, STrans *pTrans = NULL;
" reset from failed checkpoint"); int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
if (pTrans == NULL) { " reset from failed checkpoint", &pTrans);
if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return terrno; return terrno;
} }
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); if (code) {
if (code != 0) { sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
@ -79,14 +87,15 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -99,8 +108,9 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t t
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
mndKillTransImpl(pMnode, transId, ""); mndKillTransImpl(pMnode, transId, "");
SStreamObj *pStream = mndGetStreamObj(pMnode, streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { code = mndGetStreamObj(pMnode, streamId, &pStream);
if (pStream == NULL || code != 0) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST; code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid);
} else { } else {
@ -159,34 +169,39 @@ int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) {
} }
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = NULL;
if (pTrans == NULL) { int32_t code =
doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code != 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
if (code) {
return code;
}
// drop all tasks // drop all tasks
if (mndStreamSetDropActionFromList(pMnode, pTrans, pList) < 0) { if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, pList)) < 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// drop stream // drop stream
if (mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED) < 0) { if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) {
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
@ -228,10 +243,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SStreamHbMsg req = {0}; SStreamHbMsg req = {0};
SArray *pFailedChkpt = NULL; SArray *pFailedChkpt = NULL;
SArray *pOrphanTasks = NULL; SArray *pOrphanTasks = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if (suspendAllStreams(pMnode, &pReq->info) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) {
return -1; return code;
} }
} }
@ -241,8 +257,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (tDecodeStreamHbMsg(&decoder, &req) < 0) { if (tDecodeStreamHbMsg(&decoder, &req) < 0) {
tCleanupStreamHbMsg(&req); tCleanupStreamHbMsg(&req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG; code = terrno = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -257,12 +273,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { if (!validateHbMsg(execInfo.pNodeList, req.vgId)) {
mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); mError("vgId:%d not exists in nodeList buf, discarded", req.vgId);
terrno = TSDB_CODE_INVALID_MSG; code = terrno = TSDB_CODE_INVALID_MSG;
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return -1; return code;
} }
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
@ -293,11 +309,23 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
.startTs = pChkInfo->consensusTs, .startTs = pChkInfo->consensusTs,
}; };
SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId); SStreamObj *pStream = NULL;
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); code = mndGetStreamObj(pMnode, p->id.streamId, &pStream);
if (code) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
continue;
}
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
SCheckpointConsensusInfo *pInfo = NULL;
code = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks, &pInfo);
if (code == 0) {
mndAddConsensusTasks(pInfo, &cp);
} else {
mError("failed to get consensus checkpoint-info");
}
SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks);
mndAddConsensusTasks(pInfo, &cp);
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
@ -338,9 +366,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
// kill the checkpoint trans and then set all tasks status to be normal // kill the checkpoint trans and then set all tasks status to be normal
if (taosArrayGetSize(pFailedChkpt) > 0) { if (taosArrayGetSize(pFailedChkpt) > 0) {
bool allReady = true; bool allReady = true;
if (pMnode != NULL) { if (pMnode != NULL) {
SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *p = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &p);
taosArrayDestroy(p); taosArrayDestroy(p);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
} else { } else {
allReady = false; allReady = false;
} }
@ -374,7 +408,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return TSDB_CODE_SUCCESS; return terrno;
} }
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg

View File

@ -153,27 +153,30 @@ int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) {
return 0; return 0;
} }
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg) { const char *pMsg, STrans ** pTrans1) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); *pTrans1 = NULL;
if (pTrans == NULL) { terrno = 0;
STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (p == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return terrno;
} }
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName); mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, p) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT; terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno)); mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
mndTransDrop(pTrans); mndTransDrop(p);
return NULL; return terrno;
} }
terrno = 0; *pTrans1 = p;
return pTrans; return 0;
} }
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@ -272,8 +275,9 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) {
continue; continue;
} }
SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId); SStreamObj *pStream = NULL;
if (pStream != NULL) { int32_t code = mndGetStreamObj(pMnode, pTransInfo->streamId, &pStream);
if (pStream != NULL && code == 0) {
if (identicalName(pStream->sourceDb, pDBName, len)) { if (identicalName(pStream->sourceDb, pDBName, len)) {
mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb);
} else if (identicalName(pStream->targetDb, pDBName, len)) { } else if (identicalName(pStream->targetDb, pDBName, len)) {

View File

@ -28,20 +28,20 @@ struct SStreamTaskIter {
int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId); int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId);
SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) { int32_t createStreamTaskIter(SStreamObj* pStream, SStreamTaskIter** pIter) {
SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter)); *pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
if (pIter == NULL) { if (*pIter == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return terrno;
} }
pIter->level = -1; (*pIter)->level = -1;
pIter->ordinalIndex = 0; (*pIter)->ordinalIndex = 0;
pIter->pStream = pStream; (*pIter)->pStream = pStream;
pIter->totalLevel = taosArrayGetSize(pStream->tasks); (*pIter)->totalLevel = taosArrayGetSize(pStream->tasks);
pIter->pTask = NULL; (*pIter)->pTask = NULL;
return pIter; return 0;
} }
bool streamTaskIterNextTask(SStreamTaskIter* pIter) { bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
@ -72,19 +72,27 @@ bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
return false; return false;
} }
SStreamTask* streamTaskIterGetCurrent(SStreamTaskIter* pIter) { int32_t streamTaskIterGetCurrent(SStreamTaskIter* pIter, SStreamTask** pTask) {
return pIter->pTask; if (pTask) {
*pTask = pIter->pTask;
if (*pTask != NULL) {
return TSDB_CODE_SUCCESS;
}
}
return TSDB_CODE_INVALID_PARA;
} }
void destroyStreamTaskIter(SStreamTaskIter* pIter) { void destroyStreamTaskIter(SStreamTaskIter* pIter) {
taosMemoryFree(pIter); taosMemoryFree(pIter);
} }
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
int32_t replica = -1; // do the replica check int32_t replica = -1; // do the replica check
int32_t code = 0;
*allReady = true; *allReady = true;
SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry)); SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry));
@ -131,10 +139,15 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
} }
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf)); (void) epsetToStr(&entry.epset, buf, tListLen(buf));
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
}
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -145,43 +158,57 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
break; break;
} }
SNodeEntry entry = {0}; SNodeEntry entry = {.nodeId = SNODE_HANDLE};
addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port);
entry.nodeId = SNODE_HANDLE; if (code) {
sdbRelease(pSdb, pObj);
continue;
}
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf)); (void) epsetToStr(&entry.epset, buf, tListLen(buf));
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
}
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pObj); sdbRelease(pSdb, pObj);
} }
return pVgroupList; *pList = pVgroupList;
return code;
} }
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj **pStream) {
void *pIter = NULL; void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; *pStream = NULL;
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { SStreamObj *p = NULL;
if (pStream->uid == streamId) { while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&p)) != NULL) {
if (p->uid == streamId) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return pStream; *pStream = p;
return TSDB_CODE_SUCCESS;
} }
sdbRelease(pSdb, pStream); sdbRelease(pSdb, p);
} }
return NULL; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) { void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) {
STrans *pTrans = mndAcquireTrans(pMnode, transId); STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans != NULL) { if (pTrans != NULL) {
mInfo("kill active transId:%d in Db:%s", transId, pDbName); mInfo("kill active transId:%d in Db:%s", transId, pDbName);
mndKillTrans(pMnode, pTrans); int32_t code = mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans);
if (code) {
mError("failed to kill trans:%d", pTrans->id);
}
} else { } else {
mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId); mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId);
} }
@ -197,11 +224,16 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj);
if (pIter != NULL) { if (pIter != NULL) {
addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); int32_t code = addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
sdbRelease(pMnode->pSdb, pObj); sdbRelease(pMnode->pSdb, pObj);
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
*hasEpset = true; if (code) {
return TSDB_CODE_SUCCESS; *hasEpset = false;
mError("failed to set epset");
} else {
*hasEpset = true;
}
return code;
} else { } else {
mError("failed to acquire snode epset"); mError("failed to acquire snode epset");
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -223,12 +255,14 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
} }
static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) {
terrno = 0;
SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -242,31 +276,45 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { if (code != TSDB_CODE_SUCCESS || (!hasEpset)) {
terrno = code; terrno = code;
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return terrno;
} }
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return terrno;
} }
mDebug("set the resume action for trans:%d", pTrans->id); mDebug("set the resume action for trans:%d", pTrans->id);
return 0; return 0;
} }
SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); *pTask = NULL;
SStreamTask *p = NULL;
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); code = streamTaskIterGetCurrent(pIter, &p);
if (pTask->id.taskId == pId->taskId) { if (code) {
continue;
}
if (p->id.taskId == pId->taskId) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return pTask; *pTask = p;
return 0;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return NULL; return TSDB_CODE_FAILED;
} }
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) { int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
@ -280,13 +328,25 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
} }
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) { int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code || pTask == NULL) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetResumeAction(pTrans, pMnode, pTask, igUntreated);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) {
@ -303,7 +363,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq), mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -320,25 +380,38 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
} }
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&epset, buf, tListLen(buf)); (void) epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
} }
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetPauseAction(pMnode, pTrans, pTask) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetPauseAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) {
@ -348,14 +421,14 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return 0; return code;
} }
static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -366,28 +439,40 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction
terrno = code; return code;
return -1;
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
} }
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while(streamTaskIterNextTask(pIter)) { while(streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetDropAction(pMnode, pTrans, pTask) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetDropAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
@ -398,7 +483,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->nodeId); pReq->head.vgId = htonl(pTask->nodeId);
@ -409,16 +494,15 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId); int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId);
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction
terrno = code;
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
@ -427,19 +511,35 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) { int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) {
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SOrphanTask* pTask = taosArrayGet(pList, i); SOrphanTask* pTask = taosArrayGet(pList, i);
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId); int32_t code = doSetDropActionFromId(pMnode, pTrans, pTask);
doSetDropActionFromId(pMnode, pTrans, pTask); if (code != 0) {
return code;
} else {
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId);
}
} }
return 0; return 0;
} }
static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId, static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
int32_t transId) { int32_t transId) {
int32_t code = 0;
pMsg->streamId = pId->streamId; pMsg->streamId = pId->streamId;
pMsg->taskId = pId->taskId; pMsg->taskId = pId->taskId;
pMsg->transId = transId; pMsg->transId = transId;
pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo)); pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList); if (pMsg->pNodeList == NULL) {
mError("failed to prepare node list, code:out of memory");
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code == 0) {
void *p = taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
if (p == NULL) {
mError("failed to add update node list into nodeList");
}
}
} }
static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
@ -454,7 +554,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return -1; return terrno;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
@ -463,13 +563,18 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return -1; return terrno;
} }
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
tEncodeStreamTaskUpdateMsg(&encoder, &req); code = tEncodeStreamTaskUpdateMsg(&encoder, &req);
if (code == -1) {
tEncoderClear(&encoder);
taosArrayDestroy(req.pNodeList);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)buf; SMsgHead *pMsgHead = (SMsgHead *)buf;
pMsgHead->contLen = htonl(tlen); pMsgHead->contLen = htonl(tlen);
@ -487,15 +592,20 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
void *pBuf = NULL; void *pBuf = NULL;
int32_t len = 0; int32_t len = 0;
streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); int32_t code = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
if (code) {
return code;
}
doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
if (code) {
return code;
}
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { if (code != TSDB_CODE_SUCCESS || !hasEpset) {
terrno = code;
return code; return code;
} }
@ -510,16 +620,30 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
// build trans to update the epset // build trans to update the epset
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
@ -558,16 +682,30 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
} }
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
int32_t code = doSetResetAction(pMnode, pTrans, pTask); code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetResetAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
@ -581,8 +719,12 @@ static void freeTaskList(void* param) {
taosArrayDestroy(*pList); taosArrayDestroy(*pList);
} }
void mndInitExecInfo() { int32_t mndInitExecInfo() {
taosThreadMutexInit(&execInfo.lock, NULL); int32_t code = taosThreadMutexInit(&execInfo.lock, NULL);
if (code) {
return code;
}
_hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
@ -596,6 +738,7 @@ void mndInitExecInfo() {
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList); taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList); taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList);
return 0;
} }
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
@ -608,7 +751,10 @@ void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
for (int32_t j = 0; j < size; ++j) { for (int32_t j = 0; j < size; ++j) {
SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
if (pEntry->nodeId == p->nodeId) { if (pEntry->nodeId == p->nodeId) {
taosArrayPush(pValidList, p); void* px = taosArrayPush(pValidList, p);
if (px == NULL) {
mError("failed to put node into list, nodeId:%d", p->nodeId);
}
break; break;
} }
} }
@ -626,7 +772,10 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); int32_t code = taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
if (code) {
return code;
}
for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
@ -645,28 +794,45 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) { void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) {
STaskId *pId = taosArrayGet(pTaskIds, i); STaskId *pId = taosArrayGet(pTaskIds, i);
doRemoveTasks(pExecInfo, pId); int32_t code = doRemoveTasks(pExecInfo, pId);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, pId->taskId);
}
} }
} }
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexLock(&pExecNode->lock); SStreamTaskIter *pIter = NULL;
streamMutexLock(&pExecNode->lock);
// 1. remove task entries // 1. remove task entries
SStreamTaskIter *pIter = createStreamTaskIter(pStream); int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
streamMutexUnlock(&pExecNode->lock);
mError("failed to create stream task iter:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
continue;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
doRemoveTasks(pExecNode, &id); code = doRemoveTasks(pExecNode, &id);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, id.taskId);
}
} }
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
// 2. remove stream entry in consensus hash table // 2. remove stream entry in consensus hash table
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); (void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid);
taosThreadMutexUnlock(&pExecNode->lock); streamMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
} }
@ -697,7 +863,10 @@ int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) { if (!existed) {
taosArrayPush(pRemovedTasks, pId); void* p = taosArrayPush(pRemovedTasks, pId);
if (p == NULL) {
mError("failed to put task entry into remove list, taskId:0x%" PRIx64, pId->taskId);
}
} }
} }
@ -759,45 +928,64 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas
} }
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask); code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return 0; return code;
} }
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
void *pIter = NULL; void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t)); SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
int32_t code = 0;
mDebug("start to scan checkpoint report info"); mDebug("start to scan checkpoint report info");
while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) {
SArray *pList = *(SArray **)pIter; SArray *pList = *(SArray **)pIter;
STaskChkptInfo* pInfo = taosArrayGet(pList, 0); STaskChkptInfo *pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId); mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId); void* p = taosArrayPush(pDropped, &pInfo->streamId);
if (p == NULL) {
mError("failed to put stream into drop list:0x%" PRIx64, pInfo->streamId);
}
continue; continue;
} }
int32_t total = mndGetNumOfStreamTasks(pStream); int32_t total = mndGetNumOfStreamTasks(pStream);
int32_t existed = (int32_t) taosArrayGetSize(pList); int32_t existed = (int32_t)taosArrayGetSize(pList);
if (total == existed) { if (total == existed) {
mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info", mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info",
@ -805,17 +993,21 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) { if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId); void* p = taosArrayPush(pDropped, &pInfo->streamId);
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId); if (p == NULL) {
mError("failed to remove stream:0x%" PRIx64, pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
}
} else { } else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet", mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId); pInfo->streamId);
} }
break; break;
} else { } else {
mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); mDebug("stream:0x%" PRIx64 " active checkpoint trans not finished yet, wait", pInfo->streamId);
} }
} else { } else {
mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name, mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name,
@ -829,7 +1021,10 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
if (size > 0) { if (size > 0) {
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i); int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); code = taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
if (code) {
mError("failed to remove stream in buf:0x%"PRIx64, streamId);
}
} }
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
@ -854,29 +1049,30 @@ static int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStream
int32_t blen; int32_t blen;
tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code); tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code);
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
void *pBuf = taosMemoryMalloc(tlen); void *pBuf = taosMemoryMalloc(tlen);
if (pBuf == NULL) { if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
tEncodeRestoreCheckpointInfo(&encoder, &req); code = tEncodeRestoreCheckpointInfo(&encoder, &req);
tEncoderClear(&encoder);
if (code == -1) {
taosMemoryFree(pBuf);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)pBuf; SMsgHead *pMsgHead = (SMsgHead *)pBuf;
pMsgHead->contLen = htonl(tlen); pMsgHead->contLen = htonl(tlen);
pMsgHead->vgId = htonl(pTask->info.nodeId); pMsgHead->vgId = htonl(pTask->info.nodeId);
tEncoderClear(&encoder);
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
@ -898,17 +1094,28 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
char msg[128] = {0}; char msg[128] = {0};
snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId); snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId);
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg); STrans *pTrans = NULL;
if (pTrans == NULL) { int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg, &pTrans);
if (pTrans == NULL || code != 0) {
return terrno; return terrno;
} }
STaskId id = {.streamId = pStream->uid, .taskId = taskId}; STaskId id = {.streamId = pStream->uid, .taskId = taskId};
SStreamTask *pTask = mndGetStreamTask(&id, pStream); SStreamTask *pTask = NULL;
ASSERT(pTask); code = mndGetStreamTask(&id, pStream, &pTask);
if (code) {
mError("failed to get task:0x%x in stream:%s, failed to create consensus-checkpointId", taskId, pStream->name);
sdbRelease(pMnode->pSdb, pStream);
return code;
}
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid);
int32_t code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts); if (code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
}
code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts);
if (code != 0) { if (code != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -916,17 +1123,18 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
} }
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) { if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -935,10 +1143,13 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks) { int32_t mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo) {
void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); *pInfo = NULL;
if (pInfo != NULL) {
return (SCheckpointConsensusInfo*)pInfo; void* px = taosHashGet(pHash, &streamId, sizeof(streamId));
if (px != NULL) {
*pInfo = px;
return 0;
} }
SCheckpointConsensusInfo p = { SCheckpointConsensusInfo p = {
@ -947,10 +1158,14 @@ SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId,
.streamId = streamId, .streamId = streamId,
}; };
taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p)); int32_t code = taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
if (code == 0) {
void* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId)); void *pChkptInfo = (SCheckpointConsensusInfo *)taosHashGet(pHash, &streamId, sizeof(streamId));
return pChkptInfo; *pInfo = pChkptInfo;
} else {
*pInfo = NULL;
}
return code;
} }
// no matter existed or not, add the request into info list anyway, since we need to send rsp mannually // no matter existed or not, add the request into info list anyway, since we need to send rsp mannually
@ -971,11 +1186,15 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo
} }
} }
taosArrayPush(pInfo->pTaskList, &info); void *p = taosArrayPush(pInfo->pTaskList, &info);
int32_t num = taosArrayGetSize(pInfo->pTaskList); if (p == NULL) {
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64 mError("s-task:0x%x failed to put task into consensus-checkpointId list, code: out of memory", info.req.taskId);
" waiting tasks:%d", } else {
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num); int32_t num = taosArrayGetSize(pInfo->pTaskList);
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64
" waiting tasks:%d",
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num);
}
} }
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
@ -984,22 +1203,14 @@ void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
} }
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {
taosHashRemove(pHash, &streamId, sizeof(streamId)); int32_t code = taosHashRemove(pHash, &streamId, sizeof(streamId));
int32_t numOfStreams = taosHashGetSize(pHash); if (code == 0) {
mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", streamId, int32_t numOfStreams = taosHashGetSize(pHash);
numOfStreams); mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d",
return TSDB_CODE_SUCCESS; streamId, numOfStreams);
} } else {
mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list", streamId);
}
//int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) { return code;
// void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); }
// ASSERT(pInfo == NULL);
//
// SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL};
// taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
//
// SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
// ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0);
// mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId);
// return TSDB_CODE_SUCCESS;
//}

View File

@ -100,6 +100,7 @@ END:
static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg,
SSubplan *pPlan) { SSubplan *pPlan) {
int32_t code = 0;
SMqRebVgReq req = {0}; SMqRebVgReq req = {0};
int32_t code = 0; int32_t code = 0;
SEncoder encoder = {0}; SEncoder encoder = {0};
@ -640,7 +641,8 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
if (pTrans == NULL) { if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
@ -671,7 +673,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
END: END:
nodesDestroyNode((SNode *)pPlan); nodesDestroyNode((SNode *)pPlan);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static void freeRebalanceItem(void *param) { static void freeRebalanceItem(void *param) {
@ -860,6 +862,7 @@ END:
} }
static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) {
int32_t code = 0;
const char *key = rebInput->pRebInfo->key; const char *key = rebInput->pRebInfo->key;
SMqSubscribeObj *pSub = NULL; SMqSubscribeObj *pSub = NULL;
int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub); int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub);
@ -922,6 +925,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
MND_TMQ_NULL_CHECK(rebSubHash); MND_TMQ_NULL_CHECK(rebSubHash);
taosHashSetFreeFp(rebSubHash, freeRebalanceItem); taosHashSetFreeFp(rebSubHash, freeRebalanceItem);
MND_TMQ_RETURN_CHECK(mndCheckConsumer(pMsg, rebSubHash)); MND_TMQ_RETURN_CHECK(mndCheckConsumer(pMsg, rebSubHash));
@ -970,7 +974,7 @@ END:
taosHashCleanup(rebSubHash); taosHashCleanup(rebSubHash);
mndRebCntDec(); mndRebCntDec();
return code; TAOS_RETURN(code);
} }
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) {
@ -1089,9 +1093,9 @@ END:
if (code != 0) { if (code != 0) {
mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic); mError("cgroup %s on topic:%s, failed to drop", dropReq.cgroup, dropReq.topic);
return code; TAOS_RETURN(code);
} }
return TSDB_CODE_ACTION_IN_PROGRESS; TAOS_RETURN(TSDB_CODE_ACTION_IN_PROGRESS);
} }
void mndCleanupSubscribe(SMnode *pMnode) {} void mndCleanupSubscribe(SMnode *pMnode) {}
@ -1316,7 +1320,7 @@ END:
sdbRelease(pSdb, pSub); sdbRelease(pSdb, pSub);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return code; TAOS_RETURN(code);
} }
static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic, static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic,

View File

@ -123,15 +123,16 @@ _OUT:
} }
static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (pTrans->stage == TRN_STAGE_PREPARE) { if (pTrans->stage == TRN_STAGE_PREPARE) {
if (mndTransCheckConflict(pMnode, pTrans) < 0) { if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) {
mError("trans:%d, failed to validate trans conflicts.", pTrans->id); mError("trans:%d, failed to validate trans conflicts.", pTrans->id);
return -1; TAOS_RETURN(code);
} }
return mndTransValidatePrepareStage(pMnode, pTrans); return mndTransValidatePrepareStage(pMnode, pTrans);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) { static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
int32_t code = -1; int32_t code = -1;
SSdbRow *pRow = mndTransDecode(pRaw); SSdbRow *pRow = mndTransDecode(pRaw);
if (pRow == NULL) goto _OUT; if (pRow == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
pTrans = sdbGetRowObj(pRow); pTrans = sdbGetRowObj(pRow);
if (pTrans == NULL) goto _OUT; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
code = mndTransValidateImp(pMnode, pTrans); code = mndTransValidateImp(pMnode, pTrans);
@ -150,11 +159,10 @@ _OUT:
if (pTrans) mndTransDropData(pTrans); if (pTrans) mndTransDropData(pTrans);
if (pRow) taosMemoryFreeClear(pRow); if (pRow) taosMemoryFreeClear(pRow);
if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT); if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT);
return code; TAOS_RETURN(code);
} }
int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
terrno = TSDB_CODE_SUCCESS;
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SSdbRaw *pRaw = pMsg->pCont; SSdbRaw *pRaw = pMsg->pCont;
STrans *pTrans = NULL; STrans *pTrans = NULL;
@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (transId <= 0) { if (transId <= 0) {
mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq);
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OUT; goto _OUT;
} }
@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to validate requested trans since %s", transId, terrstr()); mError("trans:%d, failed to validate requested trans since %s", transId, terrstr());
code = 0; code = 0;
pMeta->code = terrno; pMeta->code = code;
goto _OUT; goto _OUT;
} }
@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) { if (code != 0) {
mError("trans:%d, failed to write to sdb since %s", transId, terrstr()); mError("trans:%d, failed to write to sdb since %s", transId, terrstr());
code = 0; code = 0;
pMeta->code = terrno; pMeta->code = code;
goto _OUT; goto _OUT;
} }
pTrans = mndAcquireTrans(pMnode, transId); pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code));
goto _OUT; goto _OUT;
} }
@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
_OUT: _OUT:
if (pTrans) mndReleaseTrans(pMnode, pTrans); if (pTrans) mndReleaseTrans(pMnode, pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) {
@ -258,7 +268,7 @@ _OUT:
mndPostMgmtCode(pMnode, code ? code : pMeta->code); mndPostMgmtCode(pMnode, code ? code : pMeta->code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return code; TAOS_RETURN(code);
} }
SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) { SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) {
@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) {
pNode->clusterId); pNode->clusterId);
} }
int32_t code = 0;
tsem_init(&pMgmt->syncSem, 0, 0); tsem_init(&pMgmt->syncSem, 0, 0);
pMgmt->sync = syncOpen(&syncInfo, true); pMgmt->sync = syncOpen(&syncInfo, true);
if (pMgmt->sync <= 0) { if (pMgmt->sync <= 0) {
mError("failed to open sync since %s", terrstr()); if (terrno != 0) code = terrno;
return -1; mError("failed to open sync since %s", tstrerror(code));
TAOS_RETURN(code);
} }
pMnode->pSdb->sync = pMgmt->sync; pMnode->pSdb->sync = pMgmt->sync;
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync); mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0; TAOS_RETURN(code);
} }
void mndCleanupSync(SMnode *pMnode) { void mndCleanupSync(SMnode *pMnode) {
@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)};
if (req.contLen <= 0) return -1; if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY;
req.pCont = rpcMallocCont(req.contLen); req.pCont = rpcMallocCont(req.contLen);
if (req.pCont == NULL) return -1; if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY;
memcpy(req.pCont, pRaw, req.contLen); memcpy(req.pCont, pRaw, req.contLen);
taosThreadMutexLock(&pMgmt->lock); taosThreadMutexLock(&pMgmt->lock);
@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosThreadMutexUnlock(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
rpcFreeCont(req.pCont); rpcFreeCont(req.pCont);
terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED);
return terrno;
} }
mInfo("trans:%d, will be proposed", transId); mInfo("trans:%d, will be proposed", transId);

View File

@ -19,21 +19,22 @@
#include "query.h" #include "query.h"
#include "qworker.h" #include "qworker.h"
SQnode *qndOpen(const SQnodeOpt *pOption) { int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
if (NULL == pQnode) { if (NULL == *pQnode) {
qError("calloc SQnode failed"); qError("calloc SQnode failed");
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pQnode->qndId = QNODE_HANDLE; (*pQnode)->qndId = QNODE_HANDLE;
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, (void **)&pQnode->pQuery, &pOption->msgCb)) { int32_t code = qWorkerInit(NODE_TYPE_QNODE, (*pQnode)->qndId, (void **)&(*pQnode)->pQuery, &pOption->msgCb);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFreeClear(pQnode); taosMemoryFreeClear(pQnode);
return NULL; return code;
} }
pQnode->msgCb = pOption->msgCb; (*pQnode)->msgCb = pOption->msgCb;
return pQnode; return TSDB_CODE_SUCCESS;
} }
void qndClose(SQnode *pQnode) { void qndClose(SQnode *pQnode) {

View File

@ -22,7 +22,6 @@ set(
# meta # meta
"src/meta/metaOpen.c" "src/meta/metaOpen.c"
"src/meta/metaIdx.c"
"src/meta/metaTable.c" "src/meta/metaTable.c"
"src/meta/metaSma.c" "src/meta/metaSma.c"
"src/meta/metaQuery.c" "src/meta/metaQuery.c"

View File

@ -155,8 +155,8 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_
SSubmitTbData* pTableData, int64_t earlyTs, const char* id); SSubmitTbData* pTableData, int64_t earlyTs, const char* id);
int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq);
#define TQ_ERR_GO_TO_END(c) \ #define TQ_ERR_GO_TO_END(c) \
do { \ do { \

View File

@ -318,7 +318,7 @@ void* tdUidStoreFree(STbUidStore* pStore);
// SMetaSnapReader ======================================== // SMetaSnapReader ========================================
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader); int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader); void metaSnapReaderClose(SMetaSnapReader** ppReader);
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData); int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
// SMetaSnapWriter ======================================== // SMetaSnapWriter ========================================
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter); int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);

View File

@ -117,91 +117,81 @@ static void freeCacheEntryFp(void* param) {
} }
int32_t metaCacheOpen(SMeta* pMeta) { int32_t metaCacheOpen(SMeta* pMeta) {
int32_t code = 0; int32_t code = 0;
SMetaCache* pCache = NULL; int32_t lino;
pCache = (SMetaCache*)taosMemoryMalloc(sizeof(SMetaCache)); pMeta->pCache = (SMetaCache*)taosMemoryCalloc(1, sizeof(SMetaCache));
if (pCache == NULL) { if (pMeta->pCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
// open entry cache // open entry cache
pCache->sEntryCache.nEntry = 0; pMeta->pCache->sEntryCache.nEntry = 0;
pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET; pMeta->pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET;
pCache->sEntryCache.aBucket = pMeta->pCache->sEntryCache.aBucket =
(SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*)); (SMetaCacheEntry**)taosMemoryCalloc(pMeta->pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*));
if (pCache->sEntryCache.aBucket == NULL) { if (pMeta->pCache->sEntryCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
// open stats cache // open stats cache
pCache->sStbStatsCache.nEntry = 0; pMeta->pCache->sStbStatsCache.nEntry = 0;
pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET; pMeta->pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET;
pCache->sStbStatsCache.aBucket = pMeta->pCache->sStbStatsCache.aBucket =
(SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*)); (SMetaStbStatsEntry**)taosMemoryCalloc(pMeta->pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*));
if (pCache->sStbStatsCache.aBucket == NULL) { if (pMeta->pCache->sStbStatsCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err2;
} }
pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); pMeta->pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->sTagFilterResCache.pUidResCache == NULL) { if (pMeta->pCache->sTagFilterResCache.pUidResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
pCache->sTagFilterResCache.accTimes = 0; pMeta->pCache->sTagFilterResCache.accTimes = 0;
pCache->sTagFilterResCache.pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->sTagFilterResCache.pTableEntry == NULL) { if (pMeta->pCache->sTagFilterResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp); taosHashSetFreeFp(pMeta->pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL); taosThreadMutexInit(&pMeta->pCache->sTagFilterResCache.lock, NULL);
pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); pMeta->pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->STbGroupResCache.pResCache == NULL) { if (pMeta->pCache->STbGroupResCache.pResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
pCache->STbGroupResCache.accTimes = 0; pMeta->pCache->STbGroupResCache.accTimes = 0;
pCache->STbGroupResCache.pTableEntry = pMeta->pCache->STbGroupResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbGroupResCache.pTableEntry == NULL) { if (pMeta->pCache->STbGroupResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); taosHashSetFreeFp(pMeta->pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); taosThreadMutexInit(&pMeta->pCache->STbGroupResCache.lock, NULL);
pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pMeta->pCache->STbFilterCache.pStb =
if (pCache->STbFilterCache.pStb == NULL) { taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
code = TSDB_CODE_OUT_OF_MEMORY; if (pMeta->pCache->STbFilterCache.pStb == NULL) {
goto _err2; TSDB_CHECK_CODE(code = terrno, lino, _exit);
} }
pCache->STbFilterCache.pStbName = pMeta->pCache->STbFilterCache.pStbName =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStbName == NULL) { if (pMeta->pCache->STbFilterCache.pStbName == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = terrno, lino, _exit);
goto _err2;
} }
pMeta->pCache = pCache; _exit:
return code; if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
_err2: metaCacheClose(pMeta);
entryCacheClose(pMeta); } else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
_err: }
taosMemoryFree(pCache);
metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
return code; return code;
} }
@ -289,8 +279,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) {
} }
} else { // insert } else { // insert
if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) { if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) {
code = metaRehashCache(pCache, 1); TAOS_UNUSED(metaRehashCache(pCache, 1));
if (code) goto _exit;
iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket; iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket;
} }
@ -328,8 +317,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sEntryCache.nEntry--; pCache->sEntryCache.nEntry--;
if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 && if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 &&
pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) { pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) {
code = metaRehashCache(pCache, 0); TAOS_UNUSED(metaRehashCache(pCache, 0));
if (code) goto _exit;
} }
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
@ -351,7 +339,9 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) {
} }
if (pEntry) { if (pEntry) {
*pInfo = pEntry->info; if (pInfo) {
*pInfo = pEntry->info;
}
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
} }
@ -415,9 +405,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) {
(*ppEntry)->info.ctbNum = pInfo->ctbNum; (*ppEntry)->info.ctbNum = pInfo->ctbNum;
} else { // insert } else { // insert
if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) { if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) {
code = metaRehashStatsCache(pCache, 1); TAOS_UNUSED(metaRehashStatsCache(pCache, 1));
if (code) goto _exit;
iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket; iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket;
} }
@ -454,8 +442,7 @@ int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sStbStatsCache.nEntry--; pCache->sStbStatsCache.nEntry--;
if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 && if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 &&
pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) { pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) {
code = metaRehashStatsCache(pCache, 0); TAOS_UNUSED(metaRehashStatsCache(pCache, 0));
if (code) goto _exit;
} }
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
@ -477,7 +464,9 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) {
} }
if (pEntry) { if (pEntry) {
*pInfo = pEntry->info; if (pInfo) {
*pInfo = pEntry->info;
}
} else { } else {
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
} }
@ -502,7 +491,9 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
// check whether it is existed in LRU cache, and remove it from linked list if not. // check whether it is existed in LRU cache, and remove it from linked list if not.
LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len); LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len);
if (pRes == NULL) { // remove the item in the linked list if (pRes == NULL) { // remove the item in the linked list
taosArrayPush(pInvalidRes, &pNode); if (taosArrayPush(pInvalidRes, &pNode) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
taosLRUCacheRelease(pCache, pRes, false); taosLRUCacheRelease(pCache, pRes, false);
} }
@ -626,7 +617,7 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
p->hitTimes = 0; p->hitTimes = 0;
tdListInit(&p->list, keyLen); tdListInit(&p->list, keyLen);
taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES); TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES));
tdListAppend(&p->list, pKey); tdListAppend(&p->list, pKey);
return 0; return 0;
} }
@ -956,9 +947,7 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) {
} }
if (tbNum && pTbArr) { if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) { for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) { TAOS_CHECK_RETURN(metaPutTbToFilterCache(pMeta, pTbArr[i], 1));
return terrno ? terrno : -1;
}
} }
} }
#else #else

View File

@ -21,7 +21,10 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) {
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
// begin a meta txn // begin a meta txn
int metaBegin(SMeta *pMeta, int8_t heap) { int32_t metaBegin(SMeta *pMeta, int8_t heap) {
int32_t code = 0;
int32_t lino;
void *(*xMalloc)(void *, size_t) = NULL; void *(*xMalloc)(void *, size_t) = NULL;
void (*xFree)(void *, void *) = NULL; void (*xFree)(void *, void *) = NULL;
void *xArg = NULL; void *xArg = NULL;
@ -36,12 +39,19 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
xArg = pMeta->pVnode->inUse; xArg = pMeta->pVnode->inUse;
} }
if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { code = tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
return -1; TSDB_CHECK_CODE(code, lino, _exit);
code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
} }
tdbCommit(pMeta->pEnv, pMeta->txn);
return 0; return 0;
} }
@ -49,20 +59,36 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; }
int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); }
int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); }
int metaPrepareAsyncCommit(SMeta *pMeta) {
// return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); int metaPrepareAsyncCommit(SMeta *pMeta) {
int code = 0; // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
int code = 0;
int32_t lino;
metaWLock(pMeta); metaWLock(pMeta);
code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn); TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn));
metaULock(pMeta); metaULock(pMeta);
code = tdbCommit(pMeta->pEnv, pMeta->txn); code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
pMeta->changed = false; pMeta->changed = false;
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
}
return code; return code;
} }
// abort the meta txn // abort the meta txn
int metaAbort(SMeta *pMeta) { int metaAbort(SMeta *pMeta) {
if (!pMeta->txn) return 0; if (!pMeta->txn) {
return 0;
}
int code = tdbAbort(pMeta->pEnv, pMeta->txn); int code = tdbAbort(pMeta->pEnv, pMeta->txn);
if (code) { if (code) {
metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno)); metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno));

View File

@ -17,159 +17,166 @@
int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) { int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) {
const SColCmprWrapper *pw = &pME->colCmpr; const SColCmprWrapper *pw = &pME->colCmpr;
if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols));
if (tEncodeI32v(pCoder, pw->version) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->version));
uDebug("encode cols:%d", pw->nCols); uDebug("encode cols:%d", pw->nCols);
for (int32_t i = 0; i < pw->nCols; i++) { for (int32_t i = 0; i < pw->nCols; i++) {
SColCmpr *p = &pw->pColCmpr[i]; SColCmpr *p = &pw->pColCmpr[i];
if (tEncodeI16v(pCoder, p->id) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI16v(pCoder, p->id));
if (tEncodeU32(pCoder, p->alg) < 0) return -1; TAOS_CHECK_RETURN(tEncodeU32(pCoder, p->alg));
} }
return 0; return 0;
} }
int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) { int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) {
SColCmprWrapper *pWrapper = &pME->colCmpr; SColCmprWrapper *pWrapper = &pME->colCmpr;
if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->nCols));
if (pWrapper->nCols == 0) { if (pWrapper->nCols == 0) {
return 0; return 0;
} }
if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->version));
uDebug("dencode cols:%d", pWrapper->nCols); uDebug("dencode cols:%d", pWrapper->nCols);
pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr)); pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr));
if (pWrapper->pColCmpr == NULL) return -1; if (pWrapper->pColCmpr == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < pWrapper->nCols; i++) { for (int i = 0; i < pWrapper->nCols; i++) {
SColCmpr *p = &pWrapper->pColCmpr[i]; SColCmpr *p = &pWrapper->pColCmpr[i];
if (tDecodeI16v(pDecoder, &p->id) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &p->id));
if (tDecodeU32(pDecoder, &p->alg) < 0) return -1; TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &p->alg));
} }
return 0; return 0;
} }
static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr, static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
SSchemaWrapper *pSchema) { SSchemaWrapper *pSchema) {
pCmpr->nCols = pSchema->nCols; pCmpr->nCols = pSchema->nCols;
pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr)); if ((pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pCmpr->nCols; i++) { for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr *pColCmpr = &pCmpr->pColCmpr[i]; SColCmpr *pColCmpr = &pCmpr->pColCmpr[i];
SSchema *pColSchema = &pSchema->pSchema[i]; SSchema *pColSchema = &pSchema->pSchema[i];
pColCmpr->id = pColSchema->colId; pColCmpr->id = pColSchema->colId;
pColCmpr->alg = createDefaultColCmprByType(pColSchema->type); pColCmpr->alg = createDefaultColCmprByType(pColSchema->type);
} }
return 0;
} }
int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1; TAOS_CHECK_RETURN(tStartEncode(pCoder));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version));
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid));
if (tEncodeI64(pCoder, pME->version) < 0) return -1; if (pME->name == NULL) {
if (tEncodeI8(pCoder, pME->type) < 0) return -1; return TSDB_CODE_INVALID_PARA;
if (tEncodeI64(pCoder, pME->uid) < 0) return -1; }
if (pME->name == NULL || tEncodeCStr(pCoder, pME->name) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name));
if (pME->type == TSDB_SUPER_TABLE) { if (pME->type == TSDB_SUPER_TABLE) {
if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor? TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags));
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow));
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) { if (TABLE_IS_ROLLUP(pME->flags)) {
if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
} }
} else if (pME->type == TSDB_CHILD_TABLE) { } else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime));
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays));
if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) { if (pME->ctbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment));
} }
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid));
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) { } else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime));
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays));
if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) { if (pME->ntbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment));
} }
if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1; TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid));
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) { } else if (pME->type == TSDB_TSMA_TABLE) {
if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma));
} else { } else {
metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type);
return TSDB_CODE_INVALID_PARA;
return -1;
} }
if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1; TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME));
tEndEncode(pCoder); tEndEncode(pCoder);
return 0; return 0;
} }
int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tStartDecode(pCoder) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(pCoder));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version));
if (tDecodeI64(pCoder, &pME->version) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type));
if (tDecodeI8(pCoder, &pME->type) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid));
if (tDecodeI64(pCoder, &pME->uid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name));
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
if (pME->type == TSDB_SUPER_TABLE) { if (pME->type == TSDB_SUPER_TABLE) {
if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor? TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags));
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow));
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) { if (TABLE_IS_ROLLUP(pME->flags)) {
if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
} }
} else if (pME->type == TSDB_CHILD_TABLE) { } else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime));
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays));
if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) { if (pME->ctbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment));
} }
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid));
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) { } else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime));
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays));
if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) { if (pME->ntbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment));
} }
if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid));
if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1; TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) { } else if (pME->type == TSDB_TSMA_TABLE) {
pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma)); pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma));
if (!pME->smaEntry.tsma) { if (!pME->smaEntry.tsma) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true));
} else { } else {
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (pME->type == TSDB_SUPER_TABLE) { if (pME->type == TSDB_SUPER_TABLE) {
if (TABLE_IS_COL_COMPRESSED(pME->flags)) { if (TABLE_IS_COL_COMPRESSED(pME->flags)) {
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) { if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
} }
} else { } else {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
TABLE_SET_COL_COMPRESSED(pME->flags); TABLE_SET_COL_COMPRESSED(pME->flags);
} }
} else if (pME->type == TSDB_NORMAL_TABLE) { } else if (pME->type == TSDB_NORMAL_TABLE) {
if (!tDecodeIsEnd(pCoder)) { if (!tDecodeIsEnd(pCoder)) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name); uDebug("set type: %d, tableName:%s", pME->type, pME->name);
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1; TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) { if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
} }
} else { } else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name); uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow); TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
} }
TABLE_SET_COL_COMPRESSED(pME->flags); TABLE_SET_COL_COMPRESSED(pME->flags);
} }

View File

@ -1,118 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef USE_INVERTED_INDEX
#include "index.h"
#endif
#include "meta.h"
struct SMetaIdx {
#ifdef USE_INVERTED_INDEX
SIndex *pIdx;
#endif
/* data */
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
};
int metaOpenIdx(SMeta *pMeta) {
#if 0
char idxDir[128]; // TODO
char * err = NULL;
rocksdb_options_t *options = rocksdb_options_create();
// TODO
sprintf(idxDir, "%s/index", pMeta->path);
if (pMeta->pCache) {
rocksdb_options_set_row_cache(options, pMeta->pCache);
}
rocksdb_options_set_create_if_missing(options, 1);
pMeta->pIdx = rocksdb_open(options, idxDir, &err);
if (pMeta->pIdx == NULL) {
// TODO: handle error
rocksdb_options_destroy(options);
return -1;
}
rocksdb_options_destroy(options);
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
#endif
return 0;
}
#ifdef BUILD_NO_CALL
void metaCloseIdx(SMeta *pMeta) { /* TODO */
#if 0
if (pMeta->pIdx) {
rocksdb_close(pMeta->pIdx);
pMeta->pIdx = NULL;
}
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexClose(pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
// return 0;
#endif
}
int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
#ifdef USE_INVERTED_INDEX
// if (pTbCfgs->type == META_CHILD_TABLE) {
// char buf[8] = {0};
// int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId;
// sprintf(buf, "%d", colId); // colname
// char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
// tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id
// tb_uid_t tuid = 0; // child table uid
// SIndexMultiTerm *terms = indexMultiTermCreate();
// SIndexTerm *term =
// indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
// indexMultiTermAdd(terms, term);
// int ret = indexPut(pMeta->pIdx->pIdx, terms);
// indexMultiTermDestroy(terms);
// return ret;
//} else {
// return DB_DONOTINDEX;
//}
#endif
// TODO
return 0;
}
int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
#ifdef USE_INVERTED_INDEX
#endif
// TODO
return 0;
}
#endif

View File

@ -39,13 +39,13 @@ static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&p
static void metaCleanup(SMeta **ppMeta); static void metaCleanup(SMeta **ppMeta);
int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
SMeta *pMeta = NULL; SMeta *pMeta = NULL;
int ret; int32_t code = 0;
int offset; int32_t lino;
char path[TSDB_FILENAME_LEN] = {0}; int32_t offset;
char path[TSDB_FILENAME_LEN] = {0};
*ppMeta = NULL; char indexFullPath[128] = {0};
// create handle // create handle
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN); vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN);
@ -53,8 +53,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR); snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR);
if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) { if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
return -1;
} }
metaInitLock(pMeta); metaInitLock(pMeta);
@ -69,163 +68,104 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path); taosMkDir(pMeta->path);
// open env // open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback, code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey); pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTbDb // open pTbDb
ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0); code = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSkmDb // open pSkmDb
ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0); code = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pUidIdx // open pUidIdx
ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0); code = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pNameIdx // open pNameIdx
ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0); code = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pCtbIdx // open pCtbIdx
ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0); code = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSuidIdx // open pSuidIdx
ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0); code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
char indexFullPath[128] = {0};
sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
taosMkDir(indexFullPath); TAOS_UNUSED(taosMkDir(indexFullPath));
SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024}; SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024};
ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0); code = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pTtlMgr ("ttlv1.idx") // open pTtlMgr ("ttlv1.idx")
char logPrefix[128] = {0}; char logPrefix[128] = {0};
sprintf(logPrefix, "vgId:%d", TD_VID(pVnode)); sprintf(logPrefix, "vgId:%d", TD_VID(pVnode));
ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold); code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open pSmaIdx // open pSmaIdx
ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0); code = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// idx table create time // idx table create time
ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0); code = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// idx num of col, normal table only // idx num of col, normal table only
ret = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0); code = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta ncol index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0); code = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0);
if (ret < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
// open index code = metaCacheOpen(pMeta);
if (metaOpenIdx(pMeta) < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
int32_t code = metaCacheOpen(pMeta); code = metaInitTbFilterCache(pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) { if (code) {
terrno = code; metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaError("vgId:%d, failed to open meta cache since %s", TD_VID(pVnode), tstrerror(terrno)); metaCleanup(&pMeta);
goto _err; *ppMeta = NULL;
} else {
metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__);
*ppMeta = pMeta;
} }
return code;
if (metaInitTbFilterCache(pMeta) != 0) {
goto _err;
}
metaDebug("vgId:%d, meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;
return 0;
_err:
metaCleanup(&pMeta);
return -1;
} }
int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) { int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
int code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SMeta *pMeta = *ppMeta; int32_t lino;
SMeta *pMeta = *ppMeta;
if (ttlMgrNeedUpgrade(pMeta->pEnv)) { if (ttlMgrNeedUpgrade(pMeta->pEnv)) {
code = metaBegin(pMeta, META_BEGIN_HEAP_OS); code = metaBegin(pMeta, META_BEGIN_HEAP_OS);
if (code < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta); code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta);
if (code < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = metaCommit(pMeta, pMeta->txn); code = metaCommit(pMeta, pMeta->txn);
if (code < 0) { TSDB_CHECK_CODE(code, lino, _exit);
metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
} }
return TSDB_CODE_SUCCESS; _exit:
if (code) {
_err: metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaCleanup(ppMeta); metaCleanup(ppMeta);
}
return code; return code;
} }
@ -235,33 +175,42 @@ int metaClose(SMeta **ppMeta) {
} }
int metaAlterCache(SMeta *pMeta, int32_t nPage) { int metaAlterCache(SMeta *pMeta, int32_t nPage) {
int32_t code = 0;
metaWLock(pMeta); metaWLock(pMeta);
code = tdbAlter(pMeta->pEnv, nPage);
if (tdbAlter(pMeta->pEnv, nPage) < 0) {
metaULock(pMeta);
return -1;
}
metaULock(pMeta); metaULock(pMeta);
return 0;
if (code) {
metaError("vgId:%d %s failed since %s", TD_VID(pMeta->pVnode), __func__, tstrerror(code));
}
return code;
} }
int32_t metaRLock(SMeta *pMeta) { int32_t metaRLock(SMeta *pMeta) {
metaTrace("meta rlock %p", &pMeta->lock); metaTrace("meta rlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockRdlock(&pMeta->lock); int32_t code = taosThreadRwlockRdlock(&pMeta->lock);
return ret; if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
} }
int32_t metaWLock(SMeta *pMeta) { int32_t metaWLock(SMeta *pMeta) {
metaTrace("meta wlock %p", &pMeta->lock); metaTrace("meta wlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockWrlock(&pMeta->lock); int32_t code = taosThreadRwlockWrlock(&pMeta->lock);
return ret; if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
} }
int32_t metaULock(SMeta *pMeta) { int32_t metaULock(SMeta *pMeta) {
metaTrace("meta ulock %p", &pMeta->lock); metaTrace("meta ulock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockUnlock(&pMeta->lock); int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
return ret; if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
} }
static void metaCleanup(SMeta **ppMeta) { static void metaCleanup(SMeta **ppMeta) {

View File

@ -1542,9 +1542,14 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t
// slow path: search TDB // slow path: search TDB
int64_t ctbNum = 0; int64_t ctbNum = 0;
int32_t colNum = 0; int32_t colNum = 0;
vnodeGetCtbNum(pVnode, uid, &ctbNum); code = vnodeGetCtbNum(pVnode, uid, &ctbNum);
vnodeGetStbColumnNum(pVnode, uid, &colNum); if (TSDB_CODE_SUCCESS == code) {
code = vnodeGetStbColumnNum(pVnode, uid, &colNum);
}
metaULock(pVnodeObj->pMeta); metaULock(pVnodeObj->pMeta);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
if (numOfTables) *numOfTables = ctbNum; if (numOfTables) *numOfTables = ctbNum;
if (numOfCols) *numOfCols = colNum; if (numOfCols) *numOfCols = colNum;

View File

@ -25,14 +25,14 @@ struct SMetaSnapReader {
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) { int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
int32_t c = 0; int32_t c = 0;
SMetaSnapReader* pReader = NULL; SMetaSnapReader* pReader = NULL;
// alloc // alloc
pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) { if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
pReader->pMeta = pMeta; pReader->pMeta = pMeta;
pReader->sver = sver; pReader->sver = sver;
@ -40,36 +40,29 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe
// impl // impl
code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL); code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
taosMemoryFree(pReader);
goto _err;
}
code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) { if (code) {
taosMemoryFree(pReader); metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
goto _err; metaSnapReaderClose(&pReader);
*ppReader = NULL;
} else {
metaInfo("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppReader = pReader;
} }
metaInfo("vgId:%d, vnode snapshot meta reader opened", TD_VID(pMeta->pVnode));
*ppReader = pReader;
return code;
_err:
metaError("vgId:%d, vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
*ppReader = NULL;
return code; return code;
} }
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) { void metaSnapReaderClose(SMetaSnapReader** ppReader) {
int32_t code = 0; if (ppReader && *ppReader) {
tdbTbcClose((*ppReader)->pTbc);
tdbTbcClose((*ppReader)->pTbc); taosMemoryFree(*ppReader);
taosMemoryFree(*ppReader); *ppReader = NULL;
*ppReader = NULL; }
return code;
} }
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
@ -106,7 +99,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
@ -122,10 +115,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
} }
_exit: _exit:
return code; if (code) {
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode),
_err: tstrerror(code));
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code)); }
return code; return code;
} }
@ -138,26 +131,30 @@ struct SMetaSnapWriter {
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) { int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SMetaSnapWriter* pWriter; SMetaSnapWriter* pWriter;
// alloc // alloc
pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) { if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
pWriter->pMeta = pMeta; pWriter->pMeta = pMeta;
pWriter->sver = sver; pWriter->sver = sver;
pWriter->ever = ever; pWriter->ever = ever;
metaBegin(pMeta, META_BEGIN_HEAP_NIL); code = metaBegin(pMeta, META_BEGIN_HEAP_NIL);
TSDB_CHECK_CODE(code, lino, _exit);
*ppWriter = pWriter; _exit:
return code; if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
_err: taosMemoryFree(pWriter);
metaError("vgId:%d, meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); *ppWriter = NULL;
*ppWriter = NULL; } else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppWriter = pWriter;
}
return code; return code;
} }
@ -189,25 +186,24 @@ _err:
int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
int32_t line = 0; int32_t lino = 0;
SMeta* pMeta = pWriter->pMeta; SMeta* pMeta = pWriter->pMeta;
SMetaEntry metaEntry = {0}; SMetaEntry metaEntry = {0};
SDecoder* pDecoder = &(SDecoder){0}; SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = metaDecodeEntry(pDecoder, &metaEntry); code = metaDecodeEntry(pDecoder, &metaEntry);
VND_CHECK_CODE(code, line, _err); TSDB_CHECK_CODE(code, lino, _exit);
code = metaHandleEntry(pMeta, &metaEntry); code = metaHandleEntry(pMeta, &metaEntry);
VND_CHECK_CODE(code, line, _err); TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
}
tDecoderClear(pDecoder); tDecoderClear(pDecoder);
return code; return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
return code;
} }
typedef struct STableInfoForChildTable { typedef struct STableInfoForChildTable {
@ -468,19 +464,17 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
return c; return c;
} }
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid){ void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
bool ret = false; bool ret = false;
SSchemaWrapper *schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1);
if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY){ if (schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true; ret = true;
} }
tDeleteSchemaWrapper(schema); tDeleteSchemaWrapper(schema);
ctx->hasPrimaryKey = ret; ctx->hasPrimaryKey = ret;
} }
bool taosXGetTablePrimaryKey(SSnapContext* ctx){ bool taosXGetTablePrimaryKey(SSnapContext* ctx) { return ctx->hasPrimaryKey; }
return ctx->hasPrimaryKey;
}
int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) { int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
int32_t ret = 0; int32_t ret = 0;

View File

@ -23,7 +23,9 @@ static int32_t rsmaRestore(SSma *pSma);
#define SMA_SET_KEEP_CFG(v, l) \ #define SMA_SET_KEEP_CFG(v, l) \
do { \ do { \
SRetention *r = &pCfg->retentions[l]; \ SRetention *r = &pCfg->retentions[l]; \
pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ int64_t keep = -1; \
convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep); \
pKeepCfg->keep2 = (int32_t)keep; \
pKeepCfg->keep0 = pKeepCfg->keep2; \ pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \ pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \ pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \
@ -60,8 +62,12 @@ static int32_t rsmaRestore(SSma *pSma);
* @return int32_t * @return int32_t
*/ */
static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) { static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) {
int32_t freqDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE); int32_t code = TSDB_CODE_SUCCESS;
int32_t keepDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE);
int64_t freqDuration = -1;
int64_t keepDuration = -1;
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t days = duration; // min int32_t days = duration; // min
if (days < freqDuration) { if (days < freqDuration) {
@ -76,8 +82,8 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
goto _exit; goto _exit;
} }
freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); code = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); code = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level
days *= (nFreqTimes > 1 ? nFreqTimes : 1); days *= (nFreqTimes > 1 ? nFreqTimes : 1);

View File

@ -335,8 +335,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
} }
if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
int64_t msInterval = int64_t msInterval = -1;
convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); TAOS_CHECK_RETURN(convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision,
TIME_UNIT_MILLISECOND, &msInterval));
pItem->maxDelay = (int32_t)msInterval; pItem->maxDelay = (int32_t)msInterval;
} else { } else {
pItem->maxDelay = (int32_t)param->maxdelay[idx]; pItem->maxDelay = (int32_t)param->maxdelay[idx];

View File

@ -69,8 +69,9 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
} }
STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg;
int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); int64_t sInterval = -1;
if (sInterval <= 0) { code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval);
if (TSDB_CODE_SUCCESS != code || 0 == sInterval) {
*days = pTsdbCfg->days; *days = pTsdbCfg->days;
goto _exit; goto _exit;
} }
@ -78,7 +79,11 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
if (records >= SMA_STORAGE_SPLIT_FACTOR) { if (records >= SMA_STORAGE_SPLIT_FACTOR) {
*days = pTsdbCfg->days; *days = pTsdbCfg->days;
} else { } else {
int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); int64_t mInterval = -1;
code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2;
if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { if (daysPerFile > SMA_STORAGE_MINUTES_MAX) {
@ -196,7 +201,12 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE}; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
code = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq);
if (code) {
smaError("failed to build create-table req, code:%d", code);
continue;
}
{ {
uint64_t groupId = pDataBlock->info.id.groupId; uint64_t groupId = pDataBlock->info.id.groupId;

View File

@ -18,8 +18,6 @@
#include "tmsg.h" #include "tmsg.h"
#include "tq.h" #include "tq.h"
#define MAX_CACHE_TABLE_INFO_NUM 10240
typedef struct STableSinkInfo { typedef struct STableSinkInfo {
uint64_t uid; uint64_t uid;
tstr name; tstr name;
@ -43,7 +41,7 @@ static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, con
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
int32_t numOfTags); int32_t numOfTags);
static SArray* createDefaultTagColName(); static int32_t createDefaultTagColName(SArray** pList);
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule); int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo); static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
@ -95,7 +93,10 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
SSingleDeleteReq req = {.startTs = skey, .endTs = ekey}; SSingleDeleteReq req = {.startTs = skey, .endTs = ekey};
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1);
taosArrayPush(deleteReq->deleteReqs, &req); void* p = taosArrayPush(deleteReq->deleteReqs, &req);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (originName) name = originName; if (originName) name = originName;
taosMemoryFreeClear(name); taosMemoryFreeClear(name);
@ -149,14 +150,20 @@ static bool tqGetTableInfo(SSHashObj* pTableInfoMap, uint64_t groupId, STableSin
static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
void* buf = NULL; void* buf = NULL;
int32_t tlen = 0; int32_t tlen = 0;
encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
if (code) {
tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code));
return code;
}
SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen}; SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen};
if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg);
if (code) {
tqError("failed to put into write-queue since %s", terrstr()); tqError("failed to put into write-queue since %s", terrstr());
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) { int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
@ -166,18 +173,36 @@ int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const
// set super table name // set super table name
SName name = {0}; SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); int32_t code = tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if (code == 0) {
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code
tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName);
}
}
pCreateTableReq->ctb.tagNum = numOfTags; pCreateTableReq->ctb.tagNum = numOfTags;
return TSDB_CODE_SUCCESS; return code;
} }
SArray* createDefaultTagColName() { int32_t createDefaultTagColName(SArray** pColNameList) {
*pColNameList = NULL;
SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN); SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; if (pTagColNameList == NULL) {
taosArrayPush(pTagColNameList, tagNameStr); return TSDB_CODE_OUT_OF_MEMORY;
return pTagColNameList; }
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
void* p = taosArrayPush(pTagColNameList, tagNameStr);
if (p == NULL) {
taosArrayDestroy(pTagColNameList);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pColNameList = pTagColNameList;
return TSDB_CODE_SUCCESS;
} }
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
@ -201,18 +226,20 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
SStreamTask* pTask, int64_t suid) { SStreamTask* pTask, int64_t suid) {
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId;
tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows); tqDebug("s-task:%s build create %d table(s) msg", id, rows);
int32_t code = 0; int32_t code = 0;
SVCreateTbBatchReq reqs = {0}; SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) { if (NULL == reqs.pArray) {
tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno)); tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno));
goto _end; goto _end;
} }
@ -222,15 +249,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1); int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags); code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
if (code) {
tqError("s-task:%s vgId:%d failed to init create table msg", id, vgId);
continue;
}
taosArrayClear(tagArray); taosArrayClear(tagArray);
if (size == 2) { if (size == 2) {
STagVal tagVal = { STagVal tagVal = {
.cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(tagArray, &tagVal); void* p = taosArrayPush(tagArray, &tagVal);
pCreateTbReq->ctb.tagName = createDefaultTagColName(); if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
if (code) {
return code;
}
} else { } else {
for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
@ -245,14 +283,19 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
} else { } else {
memcpy(&tagVal.i64, pData, pTagData->info.bytes); memcpy(&tagVal.i64, pData, pTagData->info.bytes);
} }
taosArrayPush(tagArray, &tagVal); void* p = taosArrayPush(tagArray, &tagVal);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
} }
} }
tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); code = tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
taosArrayDestroy(tagArray); taosArrayDestroy(tagArray);
tagArray = NULL; tagArray = NULL;
if (pCreateTbReq->ctb.pTag == NULL) {
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
@ -270,22 +313,34 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid, setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq); void* p = taosArrayPush(reqs.pArray, pCreateTbReq);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
STableSinkInfo* pInfo = NULL; STableSinkInfo* pInfo = NULL;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo); bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo);
if (!alreadyCached) { if (!alreadyCached) {
code = doCreateSinkInfo(pCreateTbReq->name, &pInfo); code = doCreateSinkInfo(pCreateTbReq->name, &pInfo);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr); if (code) {
tqError("vgId:%d failed to create sink tableInfo for table:%s, s-task:%s", vgId, pCreateTbReq->name, id);
continue;
}
code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, id);
if (code) {
tqError("vgId:%d failed to put sink tableInfo:%s into cache, s-task:%s", vgId, pCreateTbReq->name, id);
}
} }
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); tqDebug("s-task:%s build create table:%s msg complete", id, pCreateTbReq->name);
} }
reqs.nReqs = taosArrayGetSize(reqs.pArray); reqs.nReqs = taosArrayGetSize(reqs.pArray);
code = tqPutReqToQueue(pVnode, &reqs); code = tqPutReqToQueue(pVnode, &reqs);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqError("s-task:%s failed to send create table msg", pTask->id.idStr); tqError("s-task:%s failed to send create table msg", id);
} }
_end: _end:
@ -348,15 +403,26 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k); SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k);
if (pNewRow->ts < pOldRow->ts) { if (pNewRow->ts < pOldRow->ts) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1; j += 1;
} else if (pNewRow->ts > pOldRow->ts) { } else if (pNewRow->ts > pOldRow->ts) {
taosArrayPush(pFinal, &pOldRow); void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
} else { } else {
// check for the existance of primary key // check for the existance of primary key
if (pNewRow->numOfPKs == 0) { if (pNewRow->numOfPKs == 0) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
j += 1; j += 1;
tRowDestroy(pOldRow); tRowDestroy(pOldRow);
@ -369,7 +435,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
int32_t ret = tRowKeyCompare(&kNew, &kOld); int32_t ret = tRowKeyCompare(&kNew, &kOld);
if (ret <= 0) { if (ret <= 0) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1; j += 1;
if (ret == 0) { if (ret == 0) {
@ -377,7 +447,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
tRowDestroy(pOldRow); tRowDestroy(pOldRow);
} }
} else { } else {
taosArrayPush(pFinal, &pOldRow); void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
} }
} }
@ -386,12 +460,18 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
while (j < newLen) { while (j < newLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++); SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++);
taosArrayPush(pFinal, &pRow); void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
while (k < oldLen) { while (k < oldLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++); SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++);
taosArrayPush(pFinal, &pRow); void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
taosArrayDestroy(pNew->aRowP); taosArrayDestroy(pNew->aRowP);
@ -425,34 +505,40 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
return true; return true;
} }
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule) { SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq) {
*pReq = NULL;
SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
if (pCreateTbReq == NULL) { if (pCreateTbReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
taosArrayClear(pTagArray); taosArrayClear(pTagArray);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1); int32_t code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
if (code != 0) {
STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; return code;
taosArrayPush(pTagArray, &tagVal);
tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pCreateTbReq->ctb.tagName = createDefaultTagColName(); STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
void* p = taosArrayPush(pTagArray, &tagVal);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
return code;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
// set table name // set table name
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule); setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule);
return pCreateTbReq; *pReq = pCreateTbReq;
return code;
} }
int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) { int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) {
@ -555,7 +641,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
break; break;
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex);
if (colDataIsNull_s(pColData, j)) { if (colDataIsNull_s(pColData, j)) {
@ -566,7 +655,11 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
dataIndex++; dataIndex++;
} else { } else {
void* colData = colDataGetData(pColData, j); void* colData = colDataGetData(pColData, j);
@ -574,12 +667,18 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
SValue sv = SValue sv =
(SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SValue sv = {.type = pCol->type}; SValue sv = {.type = pCol->type};
memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
dataIndex++; dataIndex++;
} }
@ -596,7 +695,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
} }
ASSERT(pRow); ASSERT(pRow);
taosArrayPush(pTableData->aRowP, &pRow); void* p = taosArrayPush(pTableData->aRowP, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
taosArrayDestroy(pVals); taosArrayDestroy(pVals);
@ -665,6 +767,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
STableSinkInfo* pTableSinkInfo = NULL; STableSinkInfo* pTableSinkInfo = NULL;
int32_t code = 0;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo); bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo);
@ -686,7 +789,11 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} else { // this groupId has not been kept in cache yet } else { // this groupId has not been kept in cache yet
if (dstTableName[0] == 0) { if (dstTableName[0] == 0) {
memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); memset(dstTableName, 0, TSDB_TABLE_NAME_LEN);
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); code = buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
if (code) {
tqDebug("s-task:%s failed to build auto create table-name:%s, groupId:0x%" PRId64, id, dstTableName, groupId);
return code;
}
} else { } else {
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
@ -699,8 +806,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} }
} }
int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); if (code == 0) {
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName);
} else {
tqDebug("s-task:%s failed to build new sinkTableInfo, dstTable:%s", id, dstTableName);
return code;
}
} }
if (alreadyCached) { if (alreadyCached) {
@ -731,20 +843,20 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = code =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1),
&pTableData->pCreateTbReq);
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) { if (code) {
tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tstrerror(code));
tstrerror(terrno));
taosMemoryFree(pTableSinkInfo); taosMemoryFree(pTableSinkInfo);
return terrno; return code;
} }
pTableSinkInfo->uid = 0; pTableSinkInfo->uid = 0;
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} else { } else {
metaReaderClear(&mr); metaReaderClear(&mr);
@ -765,12 +877,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableSinkInfo->uid = mr.me.uid; pTableSinkInfo->uid = mr.me.uid;
metaReaderClear(&mr); metaReaderClear(&mr);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} }
} }
} }
return TDB_CODE_SUCCESS; return code;
} }
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
@ -864,14 +976,21 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) { if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq); tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id); (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id);
tbData.pCreateTbReq = NULL; tbData.pCreateTbReq = NULL;
} }
continue; continue;
} }
taosArrayPush(submitReq.aSubmitTbData, &tbData); void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqDebug("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
}
code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1);
if (code) { // failed and continue
tqDebug("vgId:%d, s-task:%s submit msg failed, data lost", vgId, id);
}
} }
} }
} else { } else {
@ -918,16 +1037,24 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) { if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq); tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id); (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id);
tbData.pCreateTbReq = NULL; tbData.pCreateTbReq = NULL;
} }
continue; continue;
} }
taosArrayPush(submitReq.aSubmitTbData, &tbData); void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqError("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
continue;
}
int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); code = taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
if (code) {
tqError("vgId:%d, s-task:%s failed to put group into index map, code:%s", vgId, id, tstrerror(code));
continue;
}
} else { } else {
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
@ -951,7 +1078,10 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
taosHashCleanup(pTableIndexMap); taosHashCleanup(pTableIndexMap);
if (hasSubmit) { if (hasSubmit) {
doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks);
if (code) { // failed and continue
tqError("vgId:%d failed to build and send submit msg", vgId);
}
} else { } else {
tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); tqDebug("vgId:%d, s-task:%s write results completed", vgId, id);
@ -989,7 +1119,11 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char
} }
int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId)); int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId));
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId); if (code == 0) {
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId);
} else {
tqError("s-task:%s failed to remove table meta from hashmap, groupId:%" PRId64, id, groupId);
}
return code; return code;
} }
@ -1019,10 +1153,14 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock*
void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
tEncodeSBatchDeleteReq(&encoder, &deleteReq); code = tEncodeSBatchDeleteReq(&encoder, &deleteReq);
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosArrayDestroy(deleteReq.deleteReqs); taosArrayDestroy(deleteReq.deleteReqs);
if (code) {
return code;
}
((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode); ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode);
SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)}; SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)};

View File

@ -37,7 +37,12 @@ int32_t tqScanWal(STQ* pTq) {
// check all tasks // check all tasks
int32_t numOfTasks = 0; int32_t numOfTasks = 0;
bool shouldIdle = true; bool shouldIdle = true;
doScanWalForAllTasks(pMeta, &shouldIdle);
int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle);
if (code) {
tqError("vgId:%d failed to start all tasks, try next time", vgId);
return code;
}
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
int32_t times = (--pMeta->scanInfo.scanCounter); int32_t times = (--pMeta->scanInfo.scanCounter);
@ -51,9 +56,13 @@ int32_t tqScanWal(STQ* pTq) {
if (times > 0) { if (times > 0) {
tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION);
tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION); code = tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION);
if (code) {
tqError("vgId:%d sched scan wal in %dms failed, ignore this failure", vgId, SCAN_WAL_IDLE_DURATION);
}
} }
return 0;
return code;
} }
typedef struct SBuildScanWalMsgParam { typedef struct SBuildScanWalMsgParam {
@ -69,28 +78,44 @@ static void doStartScanWal(void* param, void* tmrId) {
tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks, tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks,
pTq->pVnode->restored); pTq->pVnode->restored);
/*int32_t code = */ streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA);
taosMemoryFree(pParam); taosMemoryFree(pParam);
if (code) {
tqError("vgId:%d failed sched task to scan wal", vgId);
}
} }
int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam));
if (pParam == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pParam->pTq = pTq; pParam->pTq = pTq;
pParam->numOfTasks = numOfTasks; pParam->numOfTasks = numOfTasks;
tmr_h pTimer = streamTimerGetInstance(); tmr_h pTimer = NULL;
ASSERT(pTimer); code = streamTimerGetInstance(&pTimer);
if (code) {
tqError("vgId:%d failed to get tmr ctrl during sched scan wal", vgId);
return code;
}
if (pMeta->scanInfo.scanTimer == NULL) { if (pMeta->scanInfo.scanTimer == NULL) {
pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer);
} else { } else {
taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer);
if (code) {
tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration);
}
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
@ -207,7 +232,11 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) {
double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0; double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0;
qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs", qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs",
id, pTask->step2Range.minVer, maxVer, el); id, pTask->step2Range.minVer, maxVer, el);
/*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask); int32_t code = streamTaskPutTranstateIntoInputQ(pTask);
if (code) {
qError("s-task:%s failed to put trans-state into inputQ", id);
}
return true; return true;
} else { } else {
qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64 qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64
@ -290,8 +319,12 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems
break; break;
} }
} else { } else {
walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer);
code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
if (code) {
tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer);
}
break; break;
} }
} }
@ -347,18 +380,18 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX; int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name); tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems); bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if ((numOfItems > 0) || hasNewData) { if ((numOfItems > 0) || hasNewData) {
noDataInWal = false; noDataInWal = false;
@ -366,7 +399,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
taosArrayDestroy(pTaskList); taosArrayDestroy(pTaskList);
return -1; return code;
} }
} }
@ -379,5 +412,5 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
} }
taosArrayDestroy(pTaskList); taosArrayDestroy(pTaskList);
return 0; return TSDB_CODE_SUCCESS;
} }

View File

@ -160,6 +160,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
bool updated = false; bool updated = false;
int32_t code = 0;
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
@ -258,26 +259,40 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
} }
} }
// save // stream do update the nodeEp info, write it into stream meta.
if (updated) { if (updated) {
tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId); tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId);
streamMetaSaveTask(pMeta, pTask); code = streamMetaSaveTask(pMeta, pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamMetaSaveTask(pMeta, *ppHTask); code = streamMetaSaveTask(pMeta, *ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code));
}
} }
} else { } else {
tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId); tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId);
} }
streamTaskStop(pTask); code = streamTaskStop(pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamTaskStop(*ppHTask); code = streamTaskStop(*ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code));
}
} }
// keep info // keep info
streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st); streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st);
rsp.code = 0; rsp.code = TSDB_CODE_SUCCESS;
// possibly only handle the stream task. // possibly only handle the stream task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
@ -305,13 +320,16 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
#if 0 #if 0
taosMSleep(5000);// for test purpose, to trigger the leader election taosMSleep(5000);// for test purpose, to trigger the leader election
#endif #endif
tqStreamTaskStartAsync(pMeta, cb, true); code = tqStreamTaskStartAsync(pMeta, cb, true);
if (code) {
tqError("vgId:%d async start all tasks, failed, code:%s", vgId, tstrerror(code));
}
} }
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return rsp.code; return rsp.code; // always return true
} }
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -333,7 +351,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask) { if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
return -1; return -1;
@ -393,14 +411,14 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask) { if (pTask && (code == 0)) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code); code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} else { } else {
tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId);
terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST;
return terrno; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
} }
@ -408,16 +426,22 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SDecoder decoder; SDecoder decoder;
SStreamRetrieveReq req; SStreamRetrieveReq req;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req); code = tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d failed to decode retrieve msg, quit handling it", pMeta->vgId);
return code;
}
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || code != 0) {
tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
req.dstTaskId); req.dstTaskId);
tCleanupStreamRetrieveReq(&req); tCleanupStreamRetrieveReq(&req);
@ -446,6 +470,7 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamTaskCheckReq req; SStreamTaskCheckReq req;
SStreamTaskCheckRsp rsp = {0}; SStreamTaskCheckRsp rsp = {0};
@ -453,9 +478,14 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamTaskCheckReq(&decoder, &req); code = tDecodeStreamTaskCheckReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d decode check msg failed, not handle this msg", pMeta->vgId);
return code;
}
streamTaskProcessCheckMsg(pMeta, &req, &rsp); streamTaskProcessCheckMsg(pMeta, &req, &rsp);
return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId); return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId);
} }
@ -490,7 +520,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask); code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask);
if (pTask == NULL) { if ((pTask == NULL) || (code != 0)) {
return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId);
} }
@ -518,19 +548,25 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask);
if (pTask == NULL) { if (code != 0) {
tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId,
pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
if (code) {
return code;
}
{ // send checkpoint ready rsp { // send checkpoint ready rsp
SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg)); SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg));
if (pReadyRsp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReadyRsp->upstreamTaskId = req.upstreamTaskId; pReadyRsp->upstreamTaskId = req.upstreamTaskId;
pReadyRsp->upstreamNodeId = req.upstreamNodeId; pReadyRsp->upstreamNodeId = req.upstreamNodeId;
@ -606,8 +642,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
if (restored) { if (restored) {
SStreamTask* p = NULL; SStreamTask* p = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &p); code = streamMetaAcquireTask(pMeta, streamId, taskId, &p);
if ((p != NULL) && (p->info.fillHistory == 0)) { if ((p != NULL) && (code == 0) && (p->info.fillHistory == 0)) {
tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); code = tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
} }
if (p != NULL) { if (p != NULL) {
@ -631,6 +667,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
STaskId hTaskId = {0}; STaskId hTaskId = {0};
tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
@ -649,8 +686,12 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamTaskSetRemoveBackendFiles(pTask); streamTaskSetRemoveBackendFiles(pTask);
streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
if (code) {
tqError("s-task:0x%x failed to clear related fill-history info, still exists", pReq->taskId);
}
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -658,11 +699,17 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
// drop the related fill-history task firstly // drop the related fill-history task firstly
if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
}
} }
// drop the stream task now // drop the stream task now
streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); code = streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop task failed", pReq->taskId, vgId);
}
// commit the update // commit the update
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
@ -674,12 +721,13 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; return 0; // always return success
} }
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId);
@ -689,7 +737,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask != NULL && (*ppTask) != NULL) { if (ppTask != NULL && (*ppTask) != NULL) {
streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq); code = streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq);
} else { // failed to get the task. } else { // failed to get the task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
tqError( tqError(
@ -700,7 +748,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// always return success when handling the requirement issued by mnode during transaction. // always return success when handling the requirement issued by mnode during transaction.
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
@ -746,7 +794,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
if (isLeader && !tsDisableStream) { if (isLeader && !tsDisableStream) {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
streamMetaStartAllTasks(pMeta); code = streamMetaStartAllTasks(pMeta);
} else { } else {
streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId);
pMeta->startInfo.restartCount = 0; pMeta->startInfo.restartCount = 0;
@ -765,16 +813,16 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) { if (type == STREAM_EXEC_T_START_ONE_TASK) {
streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); (void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0; return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) { } else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
streamMetaStartAllTasks(pMeta); (void) streamMetaStartAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
restartStreamTasks(pMeta, isLeader); (void) restartStreamTasks(pMeta, isLeader);
return 0; return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
streamMetaStopAllTasks(pMeta); (void) streamMetaStopAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
@ -783,7 +831,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { if (pTask != NULL && (code == 0)) {
char* pStatus = NULL; char* pStatus = NULL;
if (streamTaskReadyToRun(pTask, &pStatus)) { if (streamTaskReadyToRun(pTask, &pStatus)) {
int64_t execTs = pTask->status.lastExecTs; int64_t execTs = pTask->status.lastExecTs;
@ -804,12 +852,12 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { // even in halt status, the data in inputQ must be processed if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL; char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) { if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer); pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
streamExecTask(pTask); (void) streamExecTask(pTask);
} else { } else {
int8_t status = streamTaskSetSchedStatusInactive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -829,6 +877,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskStartInfo* pStartInfo = &pMeta->startInfo;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
bool scanWal = false; bool scanWal = false;
int32_t code = 0;
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
if (pStartInfo->startAllTasks == 1) { if (pStartInfo->startAllTasks == 1) {
@ -844,8 +893,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
pStartInfo->restartCount); pStartInfo->restartCount);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); return restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER));
return TSDB_CODE_SUCCESS;
} else { } else {
if (pStartInfo->restartCount == 0) { if (pStartInfo->restartCount == 0) {
tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId); tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId);
@ -862,10 +910,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
if (scanWal && (vgId != SNODE_HANDLE)) { if (scanWal && (vgId != SNODE_HANDLE)) {
tqDebug("vgId:%d start scan wal for executing tasks", vgId); tqDebug("vgId:%d start scan wal for executing tasks", vgId);
tqScanWalAsync(pMeta->ahandle, true); code = tqScanWalAsync(pMeta->ahandle, true);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
@ -873,7 +921,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -881,7 +929,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
streamTaskClearCheckInfo(pTask, true); streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks // clear flag set during do checkpoint, and open inputQ for all upstream tasks
@ -904,7 +952,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -915,7 +963,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId); pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
@ -929,11 +977,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
pTask->id.idStr, (int32_t)pReq->downstreamTaskId); pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS); TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return code;
return TSDB_CODE_SUCCESS;
} }
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
@ -948,7 +995,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
// re-send the lost checkpoint-trigger msg to downstream task // re-send the lost checkpoint-trigger msg to downstream task
tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
(int32_t)pReq->downstreamTaskId, checkpointId, transId); (int32_t)pReq->downstreamTaskId, checkpointId, transId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_SUCCESS); TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait } else { // not send checkpoint-trigger yet, wait
int32_t recv = 0, total = 0; int32_t recv = 0, total = 0;
@ -962,7 +1009,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"sending checkpoint-source/trigger", "sending checkpoint-source/trigger",
pTask->id.idStr, recv, total); pTask->id.idStr, recv, total);
} }
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
} else { // upstream not recv the checkpoint-source/trigger till now } else { // upstream not recv the checkpoint-source/trigger till now
@ -971,12 +1018,12 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger", "upstream sending checkpoint-source/trigger",
pTask->id.idStr); pTask->id.idStr);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -984,19 +1031,19 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError( tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pRsp->taskId); pMeta->vgId, pRsp->taskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d", tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
@ -1004,7 +1051,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId); pReq->taskId);
// since task is in [STOP|DROPPING] state, it is safe to assume the pause is active // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active
@ -1018,7 +1065,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pHistoryTask = NULL; pHistoryTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask);
if (pHistoryTask == NULL) { if (pHistoryTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
", it may have been dropped already", ", it may have been dropped already",
pMeta->vgId, pTask->hTaskInfo.id.taskId); pMeta->vgId, pTask->hTaskInfo.id.taskId);
@ -1042,6 +1089,8 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
bool fromVnode) { bool fromVnode) {
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
int32_t code = 0;
if (pTask == NULL) { if (pTask == NULL) {
return -1; return -1;
} }
@ -1065,18 +1114,18 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
pTask->hTaskInfo.operatorOpen = false; pTask->hTaskInfo.operatorOpen = false;
streamStartScanHistoryAsync(pTask, igUntreated); code = streamStartScanHistoryAsync(pTask, igUntreated);
} else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
tqScanWalAsync((STQ*)handle, false); code = tqScanWalAsync((STQ*)handle, false);
} else { } else {
streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
} }
} /*else { } /*else {
ASSERT(status != TASK_STATUS__UNINIT); ASSERT(status != TASK_STATUS__UNINIT);
}*/ }*/
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return 0; return code;
} }
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) { int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) {
@ -1086,15 +1135,15 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name); tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
if (code != 0) { if (code != 0) {
@ -1104,11 +1153,11 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL; SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask) { if (pHTask && (code == 0)) {
taosThreadMutexLock(&pHTask->lock); streamMutexLock(&pHTask->lock);
SStreamTaskState p = streamTaskGetStatus(pHTask); SStreamTaskState p = streamTaskGetStatus(pHTask);
tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name); tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name);
taosThreadMutexUnlock(&pHTask->lock); streamMutexUnlock(&pHTask->lock);
code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode);
} }
@ -1139,15 +1188,15 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId); pRsp->downstreamNodeId, pRsp->downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); code = streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -1173,11 +1222,11 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId); pMeta->vgId, req.taskId);
streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); (void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return TSDB_CODE_SUCCESS; return code;
} }
// discard the rsp, since it is expired. // discard the rsp, since it is expired.
@ -1193,13 +1242,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode", tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode",
pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId); pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) { if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId); pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1215,14 +1264,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
} }
pTask->chkInfo.consensusTransId = req.transId; pTask->chkInfo.consensusTransId = req.transId;
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if (pMeta->role == NODE_ROLE_LEADER) { if (pMeta->role == NODE_ROLE_LEADER) {
/*code = */ tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId);
} else { } else {
tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr); tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }

View File

@ -294,6 +294,14 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree(p->pCurrSchema); taosMemoryFree(p->pCurrSchema);
if (p->rowKey.numOfPKs > 0) {
for (int32_t i = 0; i < p->rowKey.numOfPKs; i++) {
if (IS_VAR_DATA_TYPE(p->rowKey.pks[i].type)) {
taosMemoryFree(p->rowKey.pks[i].pData);
}
}
}
if (p->pLDataIterArray) { if (p->pLDataIterArray) {
destroySttBlockReader(p->pLDataIterArray, NULL); destroySttBlockReader(p->pLDataIterArray, NULL);
} }
@ -325,13 +333,26 @@ void* tsdbCacherowsReaderClose(void* pReader) {
return NULL; return NULL;
} }
static void freeItem(void* pItem) { static void freeItemOfRow(void* pItem) {
SLastCol* pCol = (SLastCol*)pItem; SLastCol* pCol = (SLastCol*)pItem;
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) { if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData); taosMemoryFree(pCol->colVal.value.pData);
} }
} }
static void freeItemWithPk(void* pItem) {
SLastCol* pCol = (SLastCol*)pItem;
for (int i = 0; i < pCol->rowKey.numOfPKs; i++) {
if (IS_VAR_DATA_TYPE(pCol->rowKey.pks[i].type)) {
taosMemoryFree(pCol->rowKey.pks[i].pData);
}
}
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData);
}
}
static int32_t tsdbCacheQueryReseek(void* pQHandle) { static int32_t tsdbCacheQueryReseek(void* pQHandle) {
int32_t code = 0; int32_t code = 0;
SCacheRowsReader* pReader = pQHandle; SCacheRowsReader* pReader = pQHandle;
@ -407,6 +428,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
struct STColumn* pCol = &pr->pSchema->columns[slotId]; struct STColumn* pCol = &pr->pSchema->columns[slotId];
SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
if (pr->rowKey.numOfPKs > 0) {
p.rowKey.numOfPKs = pr->rowKey.numOfPKs;
for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) {
p.rowKey.pks[j].type = pr->pkColumn.type;
if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) {
p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes);
}
}
}
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
} }
@ -420,7 +451,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
continue; continue;
} }
@ -432,7 +463,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
SLastCol* p = taosArrayGet(pLastCols, k); SLastCol* p = taosArrayGet(pLastCols, k);
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k); SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
if (pColVal->rowKey.ts > p->rowKey.ts) { if (tRowKeyCompare(&pColVal->rowKey, &p->rowKey) > 0) {
if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) {
if (!COL_VAL_IS_VALUE(&p->colVal)) { if (!COL_VAL_IS_VALUE(&p->colVal)) {
hasNotNullRow = false; hasNotNullRow = false;
@ -445,6 +476,15 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
hasRes = true; hasRes = true;
p->rowKey.ts = pColVal->rowKey.ts; p->rowKey.ts = pColVal->rowKey.ts;
for (int32_t j = 0; j < p->rowKey.numOfPKs; j++) {
if (IS_VAR_DATA_TYPE(p->rowKey.pks[j].type)) {
memcpy(p->rowKey.pks[j].pData, pColVal->rowKey.pks[j].pData, pColVal->rowKey.pks[j].nData);
p->rowKey.pks[j].nData = pColVal->rowKey.pks[j].nData;
} else {
p->rowKey.pks[j].val = pColVal->rowKey.pks[j].val;
}
}
if (k == 0) { if (k == 0) {
if (TARRAY_SIZE(pTableUidList) == 0) { if (TARRAY_SIZE(pTableUidList) == 0) {
taosArrayPush(pTableUidList, &uid); taosArrayPush(pTableUidList, &uid);
@ -483,26 +523,26 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
} }
} }
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
} }
if (hasRes) { if (hasRes) {
saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
} }
taosArrayDestroyEx(pLastCols, freeItem); taosArrayDestroyEx(pLastCols, freeItemWithPk);
} else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) {
tb_uid_t uid = pTableList[i].uid; tb_uid_t uid = pTableList[i].uid;
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype); tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
continue; continue;
} }
saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr); saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
taosArrayClearEx(pRow, freeItem); taosArrayClearEx(pRow, freeItemOfRow);
taosArrayPush(pTableUidList, &uid); taosArrayPush(pTableUidList, &uid);

View File

@ -88,33 +88,33 @@ const ESyncRole vnodeStrToRole(char *str) {
int vnodeEncodeConfig(const void *pObj, SJson *pJson) { int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; const SVnodeCfg *pCfg = (SVnodeCfg *)pObj;
if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId));
if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddStringToObject(pJson, "dbname", pCfg->dbname));
if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId));
if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage));
if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache));
if (tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast));
if (tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize));
if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf));
if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap));
if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak));
if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma));
if (tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma));
if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision));
if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update));
if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression));
if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel));
if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days));
if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows));
if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows));
if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0));
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1));
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2));
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset));
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize));
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal));
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact));
if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize));
if (pCfg->tsdbCfg.retentions[0].keep > 0) { if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1; int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) { if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -124,61 +124,67 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
} }
} }
SJson *pNodeRetentions = tjsonCreateArray(); SJson *pNodeRetentions = tjsonCreateArray();
if (pNodeRetentions == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tjsonAddItemToObject(pJson, "retentions", pNodeRetentions); tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
for (int32_t i = 0; i < nRetention; ++i) { for (int32_t i = 0; i < nRetention; ++i) {
SJson *pNodeRetention = tjsonCreateObject(); SJson *pNodeRetention = tjsonCreateObject();
const SRetention *pRetention = pCfg->tsdbCfg.retentions + i; const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq));
tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit));
tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep));
tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit); TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit));
tjsonAddItemToArray(pNodeRetentions, pNodeRetention); TAOS_CHECK_RETURN(tjsonAddItemToArray(pNodeRetentions, pNodeRetention));
} }
} }
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm));
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId));
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod));
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod));
if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod));
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize));
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize));
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level));
if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles));
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm));
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm));
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger));
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin));
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd));
if (tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange));
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod));
if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix));
if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum));
if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex));
if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion));
if (tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables));
if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables));
if (tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries));
if (tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries));
if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1;
SJson *nodeInfo = tjsonCreateArray(); SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1; if (nodeInfo == NULL) {
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; return TSDB_CODE_OUT_OF_MEMORY;
}
TAOS_CHECK_RETURN(tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo));
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId, vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion); pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
SJson *info = tjsonCreateObject(); SJson *info = tjsonCreateObject();
if (info == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort));
if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn));
if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId));
if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId));
if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)));
if (tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddItemToArray(nodeInfo, info));
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId); pNode->nodeId);
} }
@ -191,50 +197,50 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code); tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code);
if (code < 0) return -1; if (code) return code;
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1; if ((code = tjsonGetStringValue(pJson, "dbname", pCfg->dbname))) return code;
tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code); tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code); tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code); tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code); tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code); tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code); tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code); tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code); tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code); tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code); tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code); tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code); tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code); tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code); tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code); tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code); tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code); tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code); tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code); tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code); tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code); tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code);
if (code < 0) return -1; if (code) return code;
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions"); SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
int32_t nRetention = tjsonGetArraySize(pNodeRetentions); int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
if (nRetention > TSDB_RETENTION_MAX) { if (nRetention > TSDB_RETENTION_MAX) {
@ -244,52 +250,54 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i); SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i);
ASSERT(pNodeRetention != NULL); ASSERT(pNodeRetention != NULL);
tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code); tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code); tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code); tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code); tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
if (code) return code;
} }
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code); tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1; if (code) return code;
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) { if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) { if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
} else { } else {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
} }
} }
#endif #endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code); tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code); tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code); tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code); tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code); tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code); tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code); tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code); tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1; if (code) return code;
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) { if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) { if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY; return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
} else { } else {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
} }
} }
#endif #endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code); tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1; if (code) return code;
#if defined(TD_ENTERPRISE) #if defined(TD_ENTERPRISE)
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) { if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) { if (tsEncryptKey[0] == 0) {
@ -303,35 +311,35 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER; if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code); tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashChange", pCfg->hashChange, code); tjsonGetNumberValue(pJson, "hashChange", pCfg->hashChange, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code); tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code); tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code);
if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code); tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code);
if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code); tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion, code); tjsonGetNumberValue(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables, code); tjsonGetNumberValue(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables, code); tjsonGetNumberValue(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables, code); tjsonGetNumberValue(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries, code); tjsonGetNumberValue(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code); tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code);
if (code < 0) return -1; if (code) return code;
SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(nodeInfo); int arraySize = tjsonGetArraySize(nodeInfo);
@ -344,15 +352,15 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1; if (info == NULL) return -1;
tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code); tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code);
if (code < 0) return -1; if (code) return code;
tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn); tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn);
tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code); tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code);
if (code < 0) return -1; if (code) return code;
char role[10] = {0}; char role[10] = {0};
code = tjsonGetStringValue(info, "isReplica", role); code = tjsonGetStringValue(info, "isReplica", role);
if (code < 0) return -1; if (code) return code;
if (strlen(role) != 0) { if (strlen(role) != 0) {
pNode->nodeRole = vnodeStrToRole(role); pNode->nodeRole = vnodeStrToRole(role);
} else { } else {

View File

@ -125,20 +125,16 @@ int vnodeBegin(SVnode *pVnode) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// begin meta // begin meta
if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) { code = metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
// begin tsdb // begin tsdb
if (tsdbBegin(pVnode->pTsdb) < 0) { code = tsdbBegin(pVnode->pTsdb);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
// begin sma // begin sma
if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) { if (VND_IS_RSMA(pVnode)) {
code = terrno; code = smaBegin(pVnode->pSma);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -171,54 +167,41 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
} }
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile; TdFilePtr pFile = NULL;
char *data; char *data = NULL;
snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP); snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
// encode info code = vnodeEncodeInfo(pInfo, &data);
data = NULL; TSDB_CHECK_CODE(code, lino, _exit);
if (vnodeEncodeInfo(pInfo, &data) < 0) {
vError("failed to encode json info.");
return -1;
}
// save info to a vnode_tmp.json // save info to a vnode_tmp.json
pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) { if (pFile == NULL) {
vError("failed to open info file:%s for write:%s", fname, terrstr()); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
if (taosWriteFile(pFile, data, strlen(data)) < 0) { if (taosWriteFile(pFile, data, strlen(data)) < 0) {
vError("failed to write info file:%s error:%s", fname, terrstr()); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
} }
if (taosFsyncFile(pFile) < 0) { if (taosFsyncFile(pFile) < 0) {
vError("failed to fsync info file:%s error:%s", fname, terrstr()); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
} }
taosCloseFile(&pFile); _exit:
if (code) {
// free info binary vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
taosMemoryFree(data); } else {
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname,
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname, pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion);
pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion); }
return 0;
_err:
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(data); taosMemoryFree(data);
return -1; return code;
} }
int vnodeCommitInfo(const char *dir) { int vnodeCommitInfo(const char *dir) {
@ -229,8 +212,7 @@ int vnodeCommitInfo(const char *dir) {
snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP); snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
if (taosRenameFile(tfname, fname) < 0) { if (taosRenameFile(tfname, fname) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); return terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
} }
vInfo("vnode info is committed, dir:%s", dir); vInfo("vnode info is committed, dir:%s", dir);
@ -238,6 +220,8 @@ int vnodeCommitInfo(const char *dir) {
} }
int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) { int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN]; char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
char *pData = NULL; char *pData = NULL;
@ -248,44 +232,35 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
// read info // read info
pFile = taosOpenFile(fname, TD_FILE_READ); pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
return -1;
} }
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _err;
} }
pData = taosMemoryMalloc(size + 1); pData = taosMemoryMalloc(size + 1);
if (pData == NULL) { if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
if (taosReadFile(pFile, pData, size) < 0) { if (taosReadFile(pFile, pData, size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
goto _err;
} }
pData[size] = '\0'; pData[size] = '\0';
taosCloseFile(&pFile);
// decode info // decode info
if (vnodeDecodeInfo(pData, pInfo) < 0) { code = vnodeDecodeInfo(pData, pInfo);
taosMemoryFree(pData); TSDB_CHECK_CODE(code, lino, _exit);
return -1;
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
} }
taosMemoryFree(pData); taosMemoryFree(pData);
return 0;
_err:
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(pData); return code;
return -1;
} }
static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
@ -297,7 +272,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
// wait last commit task // wait last commit task
vnodeAWait(&pVnode->commitTask); vnodeAWait(&pVnode->commitTask);
if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit; code = syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg);
TSDB_CHECK_CODE(code, lino, _exit);
pVnode->state.commitTerm = pVnode->state.applyTerm; pVnode->state.commitTerm = pVnode->state.applyTerm;
@ -312,17 +288,16 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN); vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN);
vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode)); vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode));
if (vnodeSaveInfo(dir, &pInfo->info) < 0) { code = vnodeSaveInfo(dir, &pInfo->info);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdbPreCommit(pVnode->pTsdb); tsdbPreCommit(pVnode->pTsdb);
metaPrepareAsyncCommit(pVnode->pMeta); code = metaPrepareAsyncCommit(pVnode->pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
code = smaPrepareAsyncCommit(pVnode->pSma); code = smaPrepareAsyncCommit(pVnode->pSma);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pVnode->mutex); taosThreadMutexLock(&pVnode->mutex);
ASSERT(pVnode->onCommit == NULL); ASSERT(pVnode->onCommit == NULL);
@ -459,10 +434,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
} }
// commit info // commit info
if (vnodeCommitInfo(dir) < 0) { code = vnodeCommitInfo(dir);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbCommitCommit(pVnode->pTsdb); code = tsdbCommitCommit(pVnode->pTsdb);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -472,10 +445,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) { code = metaFinishCommit(pVnode->pMeta, pInfo->txn);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
}
pVnode->state.committed = pInfo->info.state.committed; pVnode->state.committed = pInfo->info.state.committed;
@ -492,7 +463,7 @@ _exit:
} else { } else {
vInfo("vgId:%d, commit end", TD_VID(pVnode)); vInfo("vgId:%d, commit end", TD_VID(pVnode));
} }
return 0; return code;
} }
bool vnodeShouldRollback(SVnode *pVnode) { bool vnodeShouldRollback(SVnode *pVnode) {
@ -514,15 +485,15 @@ void vnodeRollback(SVnode *pVnode) {
offset = strlen(tFName); offset = strlen(tFName);
snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP); snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP);
(void)taosRemoveFile(tFName); TAOS_UNUSED(taosRemoveFile(tFName));
} }
static int vnodeEncodeState(const void *pObj, SJson *pJson) { static int vnodeEncodeState(const void *pObj, SJson *pJson) {
const SVState *pState = (SVState *)pObj; const SVState *pState = (SVState *)pObj;
if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit version", pState->committed));
if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID));
if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1; TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm));
return 0; return 0;
} }
@ -532,70 +503,67 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
int32_t code; int32_t code;
tjsonGetNumberValue(pJson, "commit version", pState->committed, code); tjsonGetNumberValue(pJson, "commit version", pState->committed, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code); tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
if (code < 0) return -1; if (code) return code;
tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code); tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
if (code < 0) return -1; if (code) return code;
return 0; return 0;
} }
static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) { static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
SJson *pJson; int32_t code = 0;
char *pData; int32_t lino;
SJson *pJson = NULL;
*ppData = NULL; char *pData = NULL;
pJson = tjsonCreateObject(); pJson = tjsonCreateObject();
if (pJson == NULL) { if (pJson == NULL) {
return -1; TSDB_CHECK_CODE(code = terrno, lino, _exit);
} }
if (tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config) < 0) { code = tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
if (tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state) < 0) { code = tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
pData = tjsonToString(pJson); pData = tjsonToString(pJson);
if (pData == NULL) { if (pData == NULL) {
goto _err; TSDB_CHECK_CODE(code = terrno, lino, _exit);
} }
tjsonDelete(pJson); tjsonDelete(pJson);
*ppData = pData; _exit:
return 0; if (code) {
tjsonDelete(pJson);
_err: *ppData = NULL;
tjsonDelete(pJson); } else {
return -1; *ppData = pData;
}
return code;
} }
int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) { int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
SJson *pJson = NULL; int32_t code = 0;
int32_t lino;
SJson *pJson = NULL;
pJson = tjsonParse(pData); pJson = tjsonParse(pData);
if (pJson == NULL) { if (pJson == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
return -1; return -1;
} }
if (tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config) < 0) { code = tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
if (tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state) < 0) { code = tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
tjsonDelete(pJson); tjsonDelete(pJson);
return code;
return 0;
_err:
tjsonDelete(pJson);
return -1;
} }

View File

@ -47,7 +47,7 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol
return 0; return 0;
} }
int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
STableInfoReq infoReq = {0}; STableInfoReq infoReq = {0};
STableMetaRsp metaRsp = {0}; STableMetaRsp metaRsp = {0};
SMetaReader mer1 = {0}; SMetaReader mer1 = {0};
@ -62,15 +62,15 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req // decode req
if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit4; goto _exit4;
} }
metaRsp.dbId = pVnode->config.dbId; metaRsp.dbId = pVnode->config.dbId;
strcpy(metaRsp.tbName, infoReq.tbName); (void)strcpy(metaRsp.tbName, infoReq.tbName);
memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); (void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName); (void)sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName); code = vnodeValidateTableHash(pVnode, tableFName);
if (code) { if (code) {
goto _exit4; goto _exit4;
@ -89,7 +89,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.tuid = mer1.me.uid; metaRsp.tuid = mer1.me.uid;
if (mer1.me.type == TSDB_SUPER_TABLE) { if (mer1.me.type == TSDB_SUPER_TABLE) {
strcpy(metaRsp.stbName, mer1.me.name); (void)strcpy(metaRsp.stbName, mer1.me.name);
schema = mer1.me.stbEntry.schemaRow; schema = mer1.me.stbEntry.schemaRow;
schemaTag = mer1.me.stbEntry.schemaTag; schemaTag = mer1.me.stbEntry.schemaTag;
metaRsp.suid = mer1.me.uid; metaRsp.suid = mer1.me.uid;
@ -97,7 +97,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK); metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2; if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2;
strcpy(metaRsp.stbName, mer2.me.name); (void)strcpy(metaRsp.stbName, mer2.me.name);
metaRsp.suid = mer2.me.uid; metaRsp.suid = mer2.me.uid;
schema = mer2.me.stbEntry.schemaRow; schema = mer2.me.stbEntry.schemaRow;
schemaTag = mer2.me.stbEntry.schemaTag; schemaTag = mer2.me.stbEntry.schemaTag;
@ -114,10 +114,13 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.tversion = schemaTag.version; metaRsp.tversion = schemaTag.version;
metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags)); metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags));
metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(metaRsp.numOfColumns, sizeof(SSchemaExt)); metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(metaRsp.numOfColumns, sizeof(SSchemaExt));
if (NULL == metaRsp.pSchemas || NULL == metaRsp.pSchemaExt) {
memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) { if (schemaTag.nCols) {
memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); (void)memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
} }
if (metaRsp.pSchemaExt) { if (metaRsp.pSchemaExt) {
SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1; SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
@ -134,7 +137,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// encode and send response // encode and send response
rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
if (rspLen < 0) { if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit; goto _exit;
} }
@ -148,7 +151,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
if (rspLen < 0) {
code = terrno;
goto _exit;
}
_exit: _exit:
taosMemoryFree(metaRsp.pSchemas); taosMemoryFree(metaRsp.pSchemas);
@ -174,10 +182,10 @@ _exit4:
*pMsg = rpcMsg; *pMsg = rpcMsg;
} }
return TSDB_CODE_SUCCESS; return code;
} }
int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
STableCfgReq cfgReq = {0}; STableCfgReq cfgReq = {0};
STableCfgRsp cfgRsp = {0}; STableCfgRsp cfgRsp = {0};
SMetaReader mer1 = {0}; SMetaReader mer1 = {0};
@ -192,14 +200,14 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req // decode req
if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit; goto _exit;
} }
strcpy(cfgRsp.tbName, cfgReq.tbName); (void)strcpy(cfgRsp.tbName, cfgReq.tbName);
memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); (void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName));
sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); (void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName); code = vnodeValidateTableHash(pVnode, tableFName);
if (code) { if (code) {
goto _exit; goto _exit;
@ -222,24 +230,36 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK); metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
strcpy(cfgRsp.stbName, mer2.me.name); (void)strcpy(cfgRsp.stbName, mer2.me.name);
schema = mer2.me.stbEntry.schemaRow; schema = mer2.me.stbEntry.schemaRow;
schemaTag = mer2.me.stbEntry.schemaTag; schemaTag = mer2.me.stbEntry.schemaTag;
cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; cfgRsp.ttl = mer1.me.ctbEntry.ttlDays;
cfgRsp.commentLen = mer1.me.ctbEntry.commentLen; cfgRsp.commentLen = mer1.me.ctbEntry.commentLen;
if (mer1.me.ctbEntry.commentLen > 0) { if (mer1.me.ctbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment); cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
STag *pTag = (STag *)mer1.me.ctbEntry.pTags; STag *pTag = (STag *)mer1.me.ctbEntry.pTags;
cfgRsp.tagsLen = pTag->len; cfgRsp.tagsLen = pTag->len;
cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen); cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen);
memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen); if (NULL == cfgRsp.pTags) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen);
} else if (mer1.me.type == TSDB_NORMAL_TABLE) { } else if (mer1.me.type == TSDB_NORMAL_TABLE) {
schema = mer1.me.ntbEntry.schemaRow; schema = mer1.me.ntbEntry.schemaRow;
cfgRsp.ttl = mer1.me.ntbEntry.ttlDays; cfgRsp.ttl = mer1.me.ntbEntry.ttlDays;
cfgRsp.commentLen = mer1.me.ntbEntry.commentLen; cfgRsp.commentLen = mer1.me.ntbEntry.commentLen;
if (mer1.me.ntbEntry.commentLen > 0) { if (mer1.me.ntbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment); cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
} else { } else {
ASSERT(0); ASSERT(0);
@ -250,9 +270,13 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags)); cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags));
cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt)); cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt));
memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); if (NULL == cfgRsp.pSchemas || NULL == cfgRsp.pSchemaExt) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) { if (schemaTag.nCols) {
memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); (void)memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
} }
// if (useCompress(cfgRsp.tableType)) { // if (useCompress(cfgRsp.tableType)) {
@ -271,7 +295,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// encode and send response // encode and send response
rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp);
if (rspLen < 0) { if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG; code = terrno;
goto _exit; goto _exit;
} }
@ -285,7 +309,12 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp);
rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp);
if (rspLen < 0) {
code = terrno;
goto _exit;
}
_exit: _exit:
rpcMsg.info = pMsg->info; rpcMsg.info = pMsg->info;
@ -307,7 +336,7 @@ _exit:
tFreeSTableCfgRsp(&cfgRsp); tFreeSTableCfgRsp(&cfgRsp);
metaReaderClear(&mer2); metaReaderClear(&mer2);
metaReaderClear(&mer1); metaReaderClear(&mer1);
return TSDB_CODE_SUCCESS; return code;
} }
static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) { static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) {
@ -331,7 +360,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
void *pRsp = NULL; void *pRsp = NULL;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) {
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
qError("tDeserializeSBatchReq failed"); qError("tDeserializeSBatchReq failed");
goto _exit; goto _exit;
} }
@ -352,6 +381,10 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
for (int32_t i = 0; i < msgNum; ++i) { for (int32_t i = 0; i < msgNum; ++i) {
req = taosArrayGet(batchReq.pMsgs, i); req = taosArrayGet(batchReq.pMsgs, i);
if (req == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
reqMsg.msgType = req->msgType; reqMsg.msgType = req->msgType;
reqMsg.pCont = req->msg; reqMsg.pCont = req->msg;
@ -359,13 +392,16 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
switch (req->msgType) { switch (req->msgType) {
case TDMT_VND_TABLE_META: case TDMT_VND_TABLE_META:
vnodeGetTableMeta(pVnode, &reqMsg, false); // error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetTableMeta(pVnode, &reqMsg, false);
break; break;
case TDMT_VND_TABLE_CFG: case TDMT_VND_TABLE_CFG:
vnodeGetTableCfg(pVnode, &reqMsg, false); // error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetTableCfg(pVnode, &reqMsg, false);
break; break;
case TDMT_VND_GET_STREAM_PROGRESS: case TDMT_VND_GET_STREAM_PROGRESS:
vnodeGetStreamProgress(pVnode, &reqMsg, false); // error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetStreamProgress(pVnode, &reqMsg, false);
break; break;
default: default:
qError("invalid req msgType %d", req->msgType); qError("invalid req msgType %d", req->msgType);
@ -381,24 +417,28 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
rsp.rspCode = reqMsg.code; rsp.rspCode = reqMsg.code;
rsp.msg = reqMsg.pCont; rsp.msg = reqMsg.pCont;
taosArrayPush(batchRsp.pRsps, &rsp); if (NULL == taosArrayPush(batchRsp.pRsps, &rsp)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp); rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp);
if (rspSize < 0) { if (rspSize < 0) {
qError("tSerializeSBatchRsp failed"); qError("tSerializeSBatchRsp failed");
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _exit; goto _exit;
} }
pRsp = rpcMallocCont(rspSize); pRsp = rpcMallocCont(rspSize);
if (pRsp == NULL) { if (pRsp == NULL) {
qError("rpcMallocCont %d failed", rspSize); qError("rpcMallocCont %d failed", rspSize);
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _exit; goto _exit;
} }
if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) { if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) {
qError("tSerializeSBatchRsp %d failed", rspSize); qError("tSerializeSBatchRsp %d failed", rspSize);
code = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _exit; goto _exit;
} }
@ -501,7 +541,12 @@ int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) {
} }
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) { int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1);
if (NULL == pCur) {
qError("vnode get all table list failed");
return TSDB_CODE_FAILED;
}
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t id = metaCtbCursorNext(pCur);
@ -510,11 +555,15 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
} }
STableKeyInfo info = {uid = id}; STableKeyInfo info = {uid = id};
taosArrayPush(list, &info); if (NULL == taosArrayPush(list, &info)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseCtbCursor(pCur); metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) { int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) {
@ -522,8 +571,13 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
} }
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) { int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SVnode *pVnodeObj = pVnode; SVnode *pVnodeObj = pVnode;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1);
if (NULL == pCur) {
qError("vnode get all table list failed");
return TSDB_CODE_FAILED;
}
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t id = metaCtbCursorNext(pCur);
@ -531,14 +585,20 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
break; break;
} }
taosArrayPush(list, &id); if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseCtbCursor(pCur); metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
if (!pCur) { if (!pCur) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -550,15 +610,21 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
break; break;
} }
taosArrayPush(list, &id); if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseStbCursor(pCur); metaCloseStbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1),
void *arg) { void *arg) {
int32_t code = TSDB_CODE_SUCCESS;
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
if (!pCur) { if (!pCur) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -574,11 +640,16 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
continue; continue;
} }
taosArrayPush(list, &id); if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
} }
_exit:
metaCloseStbCursor(pCur); metaCloseStbCursor(pCur);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
@ -700,16 +771,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
*num = 0; *num = 0;
int64_t arrSize = taosArrayGetSize(suidList); int64_t arrSize = taosArrayGetSize(suidList);
int32_t code = TSDB_CODE_SUCCESS;
for (int64_t i = 0; i < arrSize; ++i) { for (int64_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
int64_t ctbNum = 0; int64_t ctbNum = 0;
int32_t numOfCols = 0; int32_t numOfCols = 0;
metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols); code = metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
*num += ctbNum * (numOfCols - 1); *num += ctbNum * (numOfCols - 1);
} }
_exit:
taosArrayDestroy(suidList); taosArrayDestroy(suidList);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -728,7 +803,11 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
} }
int64_t ctbNum = 0; int64_t ctbNum = 0;
vnodeGetCtbNum(pVnode, id, &ctbNum); int32_t code = vnodeGetCtbNum(pVnode, id, &ctbNum);
if (TSDB_CODE_SUCCESS != code) {
metaCloseStbCursor(pCur);
return code;
}
*num += ctbNum; *num += ctbNum;
} }
@ -771,14 +850,17 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
rsp.vgId = req.vgId; rsp.vgId = req.vgId;
rsp.streamId = req.streamId; rsp.streamId = req.streamId;
rspLen = tSerializeStreamProgressRsp(0, 0, &rsp); rspLen = tSerializeStreamProgressRsp(0, 0, &rsp);
if (rspLen < 0) {
code = terrno;
goto _OVER;
}
if (direct) { if (direct) {
buf = rpcMallocCont(rspLen); buf = rpcMallocCont(rspLen);
} else { } else {
buf = taosMemoryCalloc(1, rspLen); buf = taosMemoryCalloc(1, rspLen);
} }
if (!buf) { if (!buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER; goto _OVER;
} }
} }
@ -787,7 +869,11 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished); code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished);
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
tSerializeStreamProgressRsp(buf, rspLen, &rsp); rspLen = tSerializeStreamProgressRsp(buf, rspLen, &rsp);
if (rspLen < 0) {
code = terrno;
goto _OVER;
}
rpcMsg.pCont = buf; rpcMsg.pCont = buf;
buf = NULL; buf = NULL;
rpcMsg.contLen = rspLen; rpcMsg.contLen = rspLen;

View File

@ -17,20 +17,19 @@
#include "vnd.h" #include "vnd.h"
static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) { static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) {
int32_t code = -1; int32_t code = 0;
STsdbFSetPartList *pList = tsdbFSetPartListCreate(); STsdbFSetPartList *pList = tsdbFSetPartListCreate();
if (pList == NULL) { if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = terrno;
goto _out; goto _out;
} }
if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) {
terrno = TSDB_CODE_INVALID_DATA_FMT; code = tDeserializeTsdbFSetPartList(buf, bufLen, pList);
goto _out; if (code) goto _out;
}
if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) { code = tsdbFSetPartListToRangeDiff(pList, ppRanges);
goto _out; if (code) goto _out;
}
code = 0;
_out: _out:
tsdbFSetPartListDestroy(&pList); tsdbFSetPartListDestroy(&pList);
return code; return code;
@ -48,29 +47,29 @@ struct SVSnapReader {
int8_t metaDone; int8_t metaDone;
SMetaSnapReader *pMetaReader; SMetaSnapReader *pMetaReader;
// tsdb // tsdb
int8_t tsdbDone; int8_t tsdbDone;
TFileSetRangeArray *pRanges; TFileSetRangeArray *pRanges;
STsdbSnapReader *pTsdbReader; STsdbSnapReader *pTsdbReader;
// tsdb raw // tsdb raw
int8_t tsdbRAWDone; int8_t tsdbRAWDone;
STsdbSnapRAWReader *pTsdbRAWReader; STsdbSnapRAWReader *pTsdbRAWReader;
// tq // tq
int8_t tqHandleDone; int8_t tqHandleDone;
STqSnapReader *pTqSnapReader; STqSnapReader *pTqSnapReader;
int8_t tqOffsetDone; int8_t tqOffsetDone;
STqSnapReader *pTqOffsetReader; STqSnapReader *pTqOffsetReader;
int8_t tqCheckInfoDone; int8_t tqCheckInfoDone;
STqSnapReader *pTqCheckInfoReader; STqSnapReader *pTqCheckInfoReader;
// stream // stream
int8_t streamTaskDone; int8_t streamTaskDone;
SStreamTaskReader *pStreamTaskReader; SStreamTaskReader *pStreamTaskReader;
int8_t streamStateDone; int8_t streamStateDone;
SStreamStateReader *pStreamStateReader; SStreamStateReader *pStreamStateReader;
// rsma // rsma
int8_t rsmaDone; int8_t rsmaDone;
TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
SRSmaSnapReader *pRsmaReader; SRSmaSnapReader *pRsmaReader;
}; };
static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) { static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
@ -88,14 +87,15 @@ static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader,
} }
static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) { static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
int32_t code = 0;
SVnode *pVnode = pReader->pVnode; SVnode *pVnode = pReader->pVnode;
int32_t code = -1;
if (pParam->data) { if (pParam->data) {
// decode // decode
SSyncTLV *datHead = (void *)pParam->data; SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT; code = TSDB_CODE_INVALID_DATA_FMT;
terrno = code;
goto _out; goto _out;
} }
@ -116,21 +116,25 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ); ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
if (ppRanges == NULL) { if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out; goto _out;
} }
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) { code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
if (code) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr()); vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
if (code) {
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out; goto _out;
} }
} break; } break;
default: default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out; goto _out;
} }
} }
@ -147,7 +151,7 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode), vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode),
(pReader->tsdbDone ? "raw" : "normal")); (pReader->tsdbDone ? "raw" : "normal"));
} }
code = 0;
_out: _out:
return code; return code;
} }
@ -160,42 +164,43 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader)); pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) { if (pReader == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _exit;
} }
pReader->pVnode = pVnode; pReader->pVnode = pVnode;
pReader->sver = sver; pReader->sver = sver;
pReader->ever = ever; pReader->ever = ever;
// snapshot info // snapshot info
if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) { code = vnodeSnapReaderDealWithSnapInfo(pReader, pParam);
goto _err; if (code) goto _exit;
}
// open tsdb snapshot raw reader // open tsdb snapshot raw reader
if (!pReader->tsdbRAWDone) { if (!pReader->tsdbRAWDone) {
ASSERT(pReader->sver == 0); ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err; if (code) goto _exit;
} }
// check snapshot ever // check snapshot ever
SSnapshot snapshot = {0}; SSnapshot snapshot = {0};
vnodeGetSnapshot(pVnode, &snapshot); code = vnodeGetSnapshot(pVnode, &snapshot);
if (code) goto _exit;
if (ever != snapshot.lastApplyIndex) { if (ever != snapshot.lastApplyIndex) {
vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64, vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64,
TD_VID(pVnode), ever, snapshot.lastApplyIndex); TD_VID(pVnode), ever, snapshot.lastApplyIndex);
code = TSDB_CODE_SYN_INTERNAL_ERROR; code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err; goto _exit;
} }
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); _exit:
*ppReader = pReader; if (code) {
return code; vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
*ppReader = NULL;
_err: } else {
vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code)); vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = NULL; *ppReader = pReader;
}
return code; return code;
} }
@ -245,6 +250,7 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVnode *pVnode = pReader->pVnode; SVnode *pVnode = pReader->pVnode;
int32_t vgId = TD_VID(pReader->pVnode); int32_t vgId = TD_VID(pReader->pVnode);
@ -261,31 +267,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ);
if (NULL == pFile) { if (NULL == pFile) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
int64_t size; int64_t size;
if (taosFStatFile(pFile, &size, NULL) < 0) { if (taosFStatFile(pFile, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto _err; TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
} }
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto _err; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
} }
((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG; ((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG;
((SSnapDataHdr *)(*ppData))->size = size + 1; ((SSnapDataHdr *)(*ppData))->size = size + 1;
((SSnapDataHdr *)(*ppData))->data[size] = '\0'; ((SSnapDataHdr *)(*ppData))->data[size] = '\0';
if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) { if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(*ppData); taosMemoryFree(*ppData);
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto _err; TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
@ -299,20 +302,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open reader if not // open reader if not
if (pReader->pMetaReader == NULL) { if (pReader->pMetaReader == NULL) {
code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader); code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = metaSnapRead(pReader->pMetaReader, ppData); code = metaSnapRead(pReader->pMetaReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err;
if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->metaDone = 1;
goto _exit; metaSnapReaderClose(&pReader->pMetaReader);
} else {
pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader);
if (code) goto _err;
}
} }
} }
@ -322,20 +322,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbReader == NULL) { if (pReader->pTsdbReader == NULL) {
code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges, code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges,
&pReader->pTsdbReader); &pReader->pTsdbReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapRead(pReader->pTsdbReader, ppData); code = tsdbSnapRead(pReader->pTsdbReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tsdbDone = 1;
goto _exit; code = tsdbSnapReaderClose(&pReader->pTsdbReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tsdbDone = 1;
code = tsdbSnapReaderClose(&pReader->pTsdbReader);
if (code) goto _err;
}
} }
} }
@ -344,20 +341,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbRAWReader == NULL) { if (pReader->pTsdbRAWReader == NULL) {
ASSERT(pReader->sver == 0); ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader); code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData); code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tsdbRAWDone = 1;
goto _exit; code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tsdbRAWDone = 1;
code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
if (code) goto _err;
}
} }
} }
@ -365,59 +359,53 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
vInfo("vgId:%d tq transform start", vgId); vInfo("vgId:%d tq transform start", vgId);
if (!pReader->tqHandleDone) { if (!pReader->tqHandleDone) {
if (pReader->pTqSnapReader == NULL) { if (pReader->pTqSnapReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE, &pReader->pTqSnapReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE,
if (code < 0) goto _err; &pReader->pTqSnapReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapRead(pReader->pTqSnapReader, ppData); code = tqSnapRead(pReader->pTqSnapReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tqHandleDone = 1;
goto _exit; code = tqSnapReaderClose(&pReader->pTqSnapReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tqHandleDone = 1;
code = tqSnapReaderClose(&pReader->pTqSnapReader);
if (code) goto _err;
}
} }
} }
if (!pReader->tqCheckInfoDone) { if (!pReader->tqCheckInfoDone) {
if (pReader->pTqCheckInfoReader == NULL) { if (pReader->pTqCheckInfoReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO, &pReader->pTqCheckInfoReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO,
if (code < 0) goto _err; &pReader->pTqCheckInfoReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapRead(pReader->pTqCheckInfoReader, ppData); code = tqSnapRead(pReader->pTqCheckInfoReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tqCheckInfoDone = 1;
goto _exit; code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tqCheckInfoDone = 1;
code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
if (code) goto _err;
}
} }
} }
if (!pReader->tqOffsetDone) { if (!pReader->tqOffsetDone) {
if (pReader->pTqOffsetReader == NULL) { if (pReader->pTqOffsetReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET, &pReader->pTqOffsetReader); code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET,
if (code < 0) goto _err; &pReader->pTqOffsetReader);
TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapRead(pReader->pTqOffsetReader, ppData); code = tqSnapRead(pReader->pTqOffsetReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->tqOffsetDone = 1;
goto _exit; code = tqSnapReaderClose(&pReader->pTqOffsetReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->tqOffsetDone = 1;
code = tqSnapReaderClose(&pReader->pTqOffsetReader);
if (code) goto _err;
}
} }
} }
@ -426,28 +414,19 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (!pReader->streamTaskDone) { if (!pReader->streamTaskDone) {
if (pReader->pStreamTaskReader == NULL) { if (pReader->pStreamTaskReader == NULL) {
code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader); code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
vError("vgId:%d open streamtask snapshot reader failed, code:%s", vgId, tstrerror(code));
goto _err;
}
} }
code = streamTaskSnapRead(pReader->pStreamTaskReader, ppData); code = streamTaskSnapRead(pReader->pStreamTaskReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
vError("vgId:%d error happens during read data from streatask snapshot, code:%s", vgId, tstrerror(code)); if (*ppData) {
goto _err; vInfo("vgId:%d no streamTask snapshot", vgId);
goto _exit;
} else { } else {
if (*ppData) { pReader->streamTaskDone = 1;
vInfo("vgId:%d no streamTask snapshot", vgId); code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} else { pReader->pStreamTaskReader = NULL;
pReader->streamTaskDone = 1;
code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
if (code) {
goto _err;
}
pReader->pStreamTaskReader = NULL;
}
} }
} }
if (!pReader->streamStateDone) { if (!pReader->streamStateDone) {
@ -457,21 +436,18 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (code) { if (code) {
pReader->streamStateDone = 1; pReader->streamStateDone = 1;
pReader->pStreamStateReader = NULL; pReader->pStreamStateReader = NULL;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = streamStateSnapRead(pReader->pStreamStateReader, ppData); code = streamStateSnapRead(pReader->pStreamStateReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->streamStateDone = 1;
goto _exit; code = streamStateSnapReaderClose(pReader->pStreamStateReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->streamStateDone = 1; pReader->pStreamStateReader = NULL;
code = streamStateSnapReaderClose(pReader->pStreamStateReader);
if (code) goto _err;
pReader->pStreamStateReader = NULL;
}
} }
} }
@ -480,20 +456,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open if not // open if not
if (pReader->pRsmaReader == NULL) { if (pReader->pRsmaReader == NULL) {
code = rsmaSnapReaderOpen(pReader->pVnode->pSma, pReader->sver, pReader->ever, &pReader->pRsmaReader); code = rsmaSnapReaderOpen(pReader->pVnode->pSma, pReader->sver, pReader->ever, &pReader->pRsmaReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = rsmaSnapRead(pReader->pRsmaReader, ppData); code = rsmaSnapRead(pReader->pRsmaReader, ppData);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err; if (*ppData) {
goto _exit;
} else { } else {
if (*ppData) { pReader->rsmaDone = 1;
goto _exit; code = rsmaSnapReaderClose(&pReader->pRsmaReader);
} else { TSDB_CHECK_CODE(code, lino, _exit);
pReader->rsmaDone = 1;
code = rsmaSnapReaderClose(&pReader->pRsmaReader);
if (code) goto _err;
}
} }
} }
@ -501,21 +474,21 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
*nData = 0; *nData = 0;
_exit: _exit:
if (*ppData) { if (code) {
SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData); vError("vgId:%d, vnode snapshot read failed at %s:%d since %s", vgId, __FILE__, lino, tstrerror(code));
pReader->index++;
*nData = sizeof(SSnapDataHdr) + pHdr->size;
pHdr->index = pReader->index;
vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", vgId, pReader->index,
pHdr->type, *nData);
} else { } else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index); if (*ppData) {
} SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData);
return code;
_err: pReader->index++;
vError("vgId:%d, vnode snapshot read failed since %s", vgId, tstrerror(code)); *nData = sizeof(SSnapDataHdr) + pHdr->size;
pHdr->index = pReader->index;
vDebug("vgId:%d, vnode snapshot read data, index:%" PRId64 " type:%d blockLen:%d ", vgId, pReader->index,
pHdr->type, *nData);
} else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index);
}
}
return code; return code;
} }
@ -532,19 +505,19 @@ struct SVSnapWriter {
SMetaSnapWriter *pMetaSnapWriter; SMetaSnapWriter *pMetaSnapWriter;
// tsdb // tsdb
TFileSetRangeArray *pRanges; TFileSetRangeArray *pRanges;
STsdbSnapWriter *pTsdbSnapWriter; STsdbSnapWriter *pTsdbSnapWriter;
// tsdb raw // tsdb raw
STsdbSnapRAWWriter *pTsdbSnapRAWWriter; STsdbSnapRAWWriter *pTsdbSnapRAWWriter;
// tq // tq
STqSnapWriter *pTqSnapHandleWriter; STqSnapWriter *pTqSnapHandleWriter;
STqSnapWriter *pTqSnapOffsetWriter; STqSnapWriter *pTqSnapOffsetWriter;
STqSnapWriter *pTqSnapCheckInfoWriter; STqSnapWriter *pTqSnapCheckInfoWriter;
// stream // stream
SStreamTaskWriter *pStreamTaskWriter; SStreamTaskWriter *pStreamTaskWriter;
SStreamStateWriter *pStreamStateWriter; SStreamStateWriter *pStreamStateWriter;
// rsma // rsma
TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2]; TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
SRSmaSnapWriter *pRsmaSnapWriter; SRSmaSnapWriter *pRsmaSnapWriter;
}; };
TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) { TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
@ -563,18 +536,18 @@ TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t
static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) { static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
int32_t code = -1; int32_t code = 0;
int32_t lino;
if (pParam->data) { if (pParam->data) {
SSyncTLV *datHead = (void *)pParam->data; SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT; TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
goto _out;
} }
STsdbRepOpts tsdbOpts = {0}; STsdbRepOpts tsdbOpts = {0};
TFileSetRangeArray **ppRanges = NULL; TFileSetRangeArray **ppRanges = NULL;
int32_t offset = 0; int32_t offset = 0;
while (offset + sizeof(SSyncTLV) < datHead->len) { while (offset + sizeof(SSyncTLV) < datHead->len) {
SSyncTLV *subField = (void *)(datHead->val + offset); SSyncTLV *subField = (void *)(datHead->val + offset);
@ -589,30 +562,30 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP
ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ); ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
if (ppRanges == NULL) { if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
goto _out; TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out;
} }
code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) { code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr()); TSDB_CHECK_CODE(code, lino, _exit);
goto _out;
}
} break; } break;
default: default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ); vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
goto _out; TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
goto _exit;
} }
} }
vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format); vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
} }
code = 0; _exit:
_out: if (code) {
vError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
}
return code; return code;
} }
@ -634,6 +607,7 @@ static int32_t vnodeEnableBgTask(SVnode *pVnode) {
int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) { int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVSnapWriter *pWriter = NULL; SVSnapWriter *pWriter = NULL;
int64_t sver = pParam->start; int64_t sver = pParam->start;
int64_t ever = pParam->end; int64_t ever = pParam->end;
@ -644,8 +618,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
// alloc // alloc
pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter)); pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) { if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
goto _err;
} }
pWriter->pVnode = pVnode; pWriter->pVnode = pVnode;
pWriter->sver = sver; pWriter->sver = sver;
@ -655,19 +628,19 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
pWriter->commitID = ++pVnode->state.commitID; pWriter->commitID = ++pVnode->state.commitID;
// snapshot info // snapshot info
if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) { code = vnodeSnapWriterDealWithSnapInfo(pWriter, pParam);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
} else {
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
} }
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
return code;
_err:
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
return code; return code;
} }
@ -785,14 +758,13 @@ _exit:
static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
// decode info // decode info
if (vnodeDecodeInfo(pHdr->data, &pWriter->info) < 0) { code = vnodeDecodeInfo(pHdr->data, &pWriter->info);
code = TSDB_CODE_INVALID_MSG; TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
// change some value // change some value
pWriter->info.state.commitID = pWriter->commitID; pWriter->info.state.commitID = pWriter->commitID;
@ -805,10 +777,8 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_
pWriter->info.config = pVnode->config; pWriter->info.config = pVnode->config;
pWriter->info.config.vndStats = vndStats; pWriter->info.config.vndStats = vndStats;
vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId); vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId);
if (vnodeSaveInfo(dir, &pWriter->info) < 0) { code = vnodeSaveInfo(dir, &pWriter->info);
code = terrno; TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
_exit: _exit:
return code; return code;
@ -816,6 +786,7 @@ _exit:
int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0; int32_t code = 0;
int32_t lino;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
@ -824,7 +795,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pHdr->index != pWriter->index + 1) { if (pHdr->index != pWriter->index + 1) {
vError("vgId:%d, unexpected vnode snapshot msg. index:%" PRId64 ", expected index:%" PRId64, TD_VID(pVnode), vError("vgId:%d, unexpected vnode snapshot msg. index:%" PRId64 ", expected index:%" PRId64, TD_VID(pVnode),
pHdr->index, pWriter->index + 1); pHdr->index, pWriter->index + 1);
return -1; TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_MSG, lino, _exit);
} }
pWriter->index = pHdr->index; pWriter->index = pHdr->index;
@ -835,17 +806,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
switch (pHdr->type) { switch (pHdr->type) {
case SNAP_DATA_CFG: { case SNAP_DATA_CFG: {
code = vnodeSnapWriteInfo(pWriter, pData, nData); code = vnodeSnapWriteInfo(pWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_META: { case SNAP_DATA_META: {
// meta // meta
if (pWriter->pMetaSnapWriter == NULL) { if (pWriter->pMetaSnapWriter == NULL) {
code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData); code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TSDB: case SNAP_DATA_TSDB:
case SNAP_DATA_DEL: { case SNAP_DATA_DEL: {
@ -853,69 +824,69 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pTsdbSnapWriter == NULL) { if (pWriter->pTsdbSnapWriter == NULL) {
code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges, code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges,
&pWriter->pTsdbSnapWriter); &pWriter->pTsdbSnapWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr); code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_RAW: { case SNAP_DATA_RAW: {
// tsdb // tsdb
if (pWriter->pTsdbSnapRAWWriter == NULL) { if (pWriter->pTsdbSnapRAWWriter == NULL) {
ASSERT(pWriter->sver == 0); ASSERT(pWriter->sver == 0);
code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter); code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr); code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TQ_HANDLE: { case SNAP_DATA_TQ_HANDLE: {
// tq handle // tq handle
if (pWriter->pTqSnapHandleWriter == NULL) { if (pWriter->pTqSnapHandleWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData); code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TQ_CHECKINFO: { case SNAP_DATA_TQ_CHECKINFO: {
// tq checkinfo // tq checkinfo
if (pWriter->pTqSnapCheckInfoWriter == NULL) { if (pWriter->pTqSnapCheckInfoWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData); code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_TQ_OFFSET: { case SNAP_DATA_TQ_OFFSET: {
// tq offset // tq offset
if (pWriter->pTqSnapOffsetWriter == NULL) { if (pWriter->pTqSnapOffsetWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter); code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData); code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_STREAM_TASK: case SNAP_DATA_STREAM_TASK:
case SNAP_DATA_STREAM_TASK_CHECKPOINT: { case SNAP_DATA_STREAM_TASK_CHECKPOINT: {
if (pWriter->pStreamTaskWriter == NULL) { if (pWriter->pStreamTaskWriter == NULL) {
code = streamTaskSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamTaskWriter); code = streamTaskSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamTaskWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData); code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_STREAM_STATE_BACKEND: { case SNAP_DATA_STREAM_STATE_BACKEND: {
if (pWriter->pStreamStateWriter == NULL) { if (pWriter->pStreamStateWriter == NULL) {
code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter); code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = streamStateSnapWrite(pWriter->pStreamStateWriter, pData, nData); code = streamStateSnapWrite(pWriter->pStreamStateWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
case SNAP_DATA_RSMA1: case SNAP_DATA_RSMA1:
@ -925,20 +896,19 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pRsmaSnapWriter == NULL) { if (pWriter->pRsmaSnapWriter == NULL) {
code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges, code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges,
&pWriter->pRsmaSnapWriter); &pWriter->pRsmaSnapWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData); code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} break; } break;
default: default:
break; break;
} }
_exit: _exit:
return code; if (code) {
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode),
_err: tstrerror(code), pHdr->index, pHdr->type, nData);
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), }
tstrerror(code), pHdr->index, pHdr->type, nData);
return code; return code;
} }

View File

@ -387,7 +387,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (msgcb == NULL || msgcb->putToQueueFp == NULL) { if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return -1; return TSDB_CODE_INVALID_PARA;
} }
int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg); int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
@ -400,13 +400,13 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (pMsg == NULL || pMsg->pCont == NULL) { if (pMsg == NULL || pMsg->pCont == NULL) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
if (msgcb == NULL || msgcb->putToQueueFp == NULL) { if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL; pMsg->pCont = NULL;
return -1; return TSDB_CODE_INVALID_PARA;
} }
int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
@ -485,8 +485,7 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta
static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader); return vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
return code;
} }
static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) { static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
@ -496,8 +495,7 @@ static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len); return vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
return code;
} }
static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) { static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
@ -514,8 +512,7 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
} }
} while (true); } while (true);
int32_t code = vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter); return vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
return code;
} }
static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
@ -580,7 +577,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
if (pMeta->startInfo.startAllTasks == 1) { if (pMeta->startInfo.startAllTasks == 1) {
pMeta->startInfo.restartCount += 1; pMeta->startInfo.restartCount += 1;
vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId,
pMeta->startInfo.restartCount); pMeta->startInfo.restartCount);
} else { } else {
pMeta->startInfo.startAllTasks = 1; pMeta->startInfo.startAllTasks = 1;
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -636,7 +633,7 @@ static void vnodeBecomeLeader(const SSyncFSM *pFsm) {
} }
} }
static void vnodeBecomeAssignedLeader(const SSyncFSM* pFsm) { static void vnodeBecomeAssignedLeader(const SSyncFSM *pFsm) {
SVnode *pVnode = pFsm->data; SVnode *pVnode = pFsm->data;
vDebug("vgId:%d, become assigned leader", pVnode->config.vgId); vDebug("vgId:%d, become assigned leader", pVnode->config.vgId);
if (pVnode->pTq) { if (pVnode->pTq) {
@ -662,12 +659,16 @@ static int32_t vnodeApplyQueueItems(const SSyncFSM *pFsm) {
int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE);
return itemSize; return itemSize;
} else { } else {
return -1; return TSDB_CODE_INVALID_PARA;
} }
} }
static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
if (pFsm == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pFsm->data = pVnode; pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitMsg; pFsm->FpCommitCb = vnodeSyncCommitMsg;
pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex; pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex;
@ -724,7 +725,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
pVnode->sync = syncOpen(&syncInfo, vnodeVersion); pVnode->sync = syncOpen(&syncInfo, vnodeVersion);
if (pVnode->sync <= 0) { if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr()); vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
return -1; return terrno;
} }
return 0; return 0;
@ -732,9 +733,10 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
int32_t vnodeSyncStart(SVnode *pVnode) { int32_t vnodeSyncStart(SVnode *pVnode) {
vInfo("vgId:%d, start sync", pVnode->config.vgId); vInfo("vgId:%d, start sync", pVnode->config.vgId);
if (syncStart(pVnode->sync) < 0) { int32_t code = syncStart(pVnode->sync);
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr()); if (code) {
return -1; vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, tstrerror(code));
return code;
} }
return 0; return 0;
} }

View File

@ -176,7 +176,14 @@ typedef struct SExplainCtx {
#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
#define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column")) #define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column"))
#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) #define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p, _r) \
do { \
if ((_u) == 'n' || (_u) == 'y') { \
_r = (_t); \
} else { \
code = convertTimeFromPrecisionToUnit(_t, _p, _u, &_r); \
} \
} while(0)
#define EXPLAIN_ROW_NEW(level, ...) \ #define EXPLAIN_ROW_NEW(level, ...) \
do { \ do { \

View File

@ -55,7 +55,7 @@ char* qExplainGetAsofOpStr(int32_t opType) {
case OP_TYPE_LOWER_EQUAL: case OP_TYPE_LOWER_EQUAL:
return "<="; return "<=";
case OP_TYPE_EQUAL: case OP_TYPE_EQUAL:
return "="; return "=";
default: default:
return "UNKNOWN"; return "UNKNOWN";
} }
@ -192,13 +192,13 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
if (0 == group->physiPlanExecIdx) { if (0 == group->physiPlanExecIdx) {
group->nodeIdx = 0; group->nodeIdx = 0;
} }
rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++);
if (group->physiPlanExecIdx >= rsp->numOfPlans) { if (group->physiPlanExecIdx >= rsp->numOfPlans) {
qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno; if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno;
} else { } else {
for (int32_t i = 0; i < group->nodeNum; ++i) { for (int32_t i = 0; i < group->nodeNum; ++i) {
@ -338,7 +338,7 @@ static char* qExplainGetScanMode(STableScanPhysiNode* pScan) {
isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname; isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname;
if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) {
return "seq_grp_order"; return "seq_grp_order";
} }
if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) {
return "grp_order"; return "grp_order";
@ -416,7 +416,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -581,7 +581,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -621,7 +621,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False");
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pPrjNode->node.pConditions) { if (pPrjNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
@ -629,7 +629,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -689,7 +689,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin); EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
@ -705,7 +705,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) { if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) {
@ -733,7 +733,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
if (NULL != pJoinNode->pColOnCond) { if (NULL != pJoinNode->pColOnCond) {
@ -741,7 +741,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
@ -831,7 +831,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -844,7 +844,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId);
QRY_ERR_RET(TSDB_CODE_APP_ERROR); QRY_ERR_RET(TSDB_CODE_APP_ERROR);
} }
nodeNum += group->nodeNum; nodeNum += group->nodeNum;
} }
@ -876,7 +876,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) { for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) {
@ -956,7 +956,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -991,10 +991,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1043,10 +1050,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1123,7 +1137,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1163,7 +1177,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1205,7 +1219,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1247,7 +1261,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1340,7 +1354,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1381,7 +1395,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1422,7 +1436,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1464,7 +1478,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1535,7 +1549,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1563,10 +1577,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1577,7 +1598,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1644,7 +1665,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1669,7 +1690,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE, QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
@ -1722,7 +1743,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
if (pJoinNode->pPrimKeyCond) { if (pJoinNode->pPrimKeyCond) {
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pColEqCond) { if (pJoinNode->pColEqCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1730,7 +1751,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pTagEqCond) { if (pJoinNode->pTagEqCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1738,7 +1759,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pFullOnCond) { if (pJoinNode->pFullOnCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1746,14 +1767,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
} }
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->timeRangeTarget) { if (pJoinNode->timeRangeTarget) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
@ -2124,7 +2145,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_ERRNO(code); TAOS_CHECK_ERRNO(code);
} }
groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum);
taosWUnLockLatch(&group->lock); taosWUnLockLatch(&group->lock);

View File

@ -432,6 +432,15 @@ typedef struct SMJoinOperatorInfo {
} \ } \
} while (0) } while (0)
#define MJ_RET(c) \
do { \
int32_t _code = c; \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
} \
return _code; \
} while (0)
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin); void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin);
@ -459,7 +468,7 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable); int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable); int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable);
int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp); int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp);
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build); int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont);
int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond); int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond);
int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx); int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx);
int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp); int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp);

View File

@ -53,7 +53,7 @@ typedef struct SDataDeleterHandle {
TdThreadMutex mutex; TdThreadMutex mutex;
} SDataDeleterHandle; } SDataDeleterHandle;
static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) { static int32_t toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) {
int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots); int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots);
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
@ -65,14 +65,23 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
pBuf->useSize = sizeof(SDataCacheEntry); pBuf->useSize = sizeof(SDataCacheEntry);
SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0);
if (NULL == pColRes) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1); SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1);
if (NULL == pColSKey) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2); SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2);
if (NULL == pColEKey) {
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SDeleterRes* pRes = (SDeleterRes*)pEntry->data; SDeleterRes* pRes = (SDeleterRes*)pEntry->data;
pRes->suid = pHandle->pParam->suid; pRes->suid = pHandle->pParam->suid;
pRes->uidList = pHandle->pParam->pUidList; pRes->uidList = pHandle->pParam->pUidList;
strcpy(pRes->tableName, pHandle->pDeleter->tableFName); TAOS_STRCPY(pRes->tableName, pHandle->pDeleter->tableFName);
strcpy(pRes->tsColName, pHandle->pDeleter->tsColName); TAOS_STRCPY(pRes->tsColName, pHandle->pDeleter->tsColName);
pRes->affectedRows = *(int64_t*)pColRes->pData; pRes->affectedRows = *(int64_t*)pColRes->pData;
if (pRes->affectedRows) { if (pRes->affectedRows) {
@ -88,16 +97,18 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp
pBuf->useSize += pEntry->dataLen; pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); (void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); (void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
return TSDB_CODE_SUCCESS;
} }
static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) { static int32_t allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) {
uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery; uint32_t capacity = pDeleter->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) { if (taosQueueItemSize(pDeleter->pDataBlocks) > capacity) {
qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity, qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
taosQueueItemSize(pDeleter->pDataBlocks)); taosQueueItemSize(pDeleter->pDataBlocks));
return false; return TSDB_CODE_OUT_OF_MEMORY;
} }
pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes); pBuf->allocSize = sizeof(SDataCacheEntry) + sizeof(SDeleterRes);
@ -105,55 +116,66 @@ static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDa
pBuf->pData = taosMemoryMalloc(pBuf->allocSize); pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
if (pBuf->pData == NULL) { if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno));
return terrno;
} }
return NULL != pBuf->pData; return TSDB_CODE_SUCCESS;
} }
static int32_t updateStatus(SDataDeleterHandle* pDeleter) { static int32_t updateStatus(SDataDeleterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex); (void)taosThreadMutexLock(&pDeleter->mutex);
int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks); int32_t blockNums = taosQueueItemSize(pDeleter->pDataBlocks);
int32_t status = int32_t status =
(0 == blockNums ? DS_BUF_EMPTY (0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); : (blockNums < pDeleter->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDeleter->status = status; pDeleter->status = status;
taosThreadMutexUnlock(&pDeleter->mutex); (void)taosThreadMutexUnlock(&pDeleter->mutex);
return status; return status;
} }
static int32_t getStatus(SDataDeleterHandle* pDeleter) { static int32_t getStatus(SDataDeleterHandle* pDeleter) {
taosThreadMutexLock(&pDeleter->mutex); (void)taosThreadMutexLock(&pDeleter->mutex);
int32_t status = pDeleter->status; int32_t status = pDeleter->status;
taosThreadMutexUnlock(&pDeleter->mutex); (void)taosThreadMutexUnlock(&pDeleter->mutex);
return status; return status;
} }
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
SDataDeleterBuf* pBuf; SDataDeleterBuf* pBuf = NULL;
int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf); int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) { if (code) {
return code; return code;
} }
if (!allocBuf(pDeleter, pInput, pBuf)) { code = allocBuf(pDeleter, pInput, pBuf);
if (code) {
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
toDataCacheEntry(pDeleter, pInput, pBuf); QRY_ERR_JRET(toDataCacheEntry(pDeleter, pInput, pBuf));
taosWriteQitem(pDeleter->pDataBlocks, pBuf); QRY_ERR_JRET(taosWriteQitem(pDeleter->pDataBlocks, pBuf));
*pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false); *pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return:
taosFreeQitem(pBuf);
return code;
} }
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
taosThreadMutexLock(&pDeleter->mutex); (void)taosThreadMutexLock(&pDeleter->mutex);
pDeleter->queryEnd = true; pDeleter->queryEnd = true;
pDeleter->useconds = useconds; pDeleter->useconds = useconds;
taosThreadMutexUnlock(&pDeleter->mutex); (void)taosThreadMutexUnlock(&pDeleter->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
@ -165,9 +187,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
} }
SDataDeleterBuf* pBuf = NULL; SDataDeleterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); (void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) { if (pBuf != NULL) {
memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); TAOS_MEMCPY(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf));
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
} }
@ -192,35 +214,35 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
} }
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData); SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
pDeleter->pParam->pUidList = NULL; pDeleter->pParam->pUidList = NULL;
pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols; pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed; pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); (void)atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDeleter); pOutput->bufStatus = updateStatus(pDeleter);
taosThreadMutexLock(&pDeleter->mutex); (void)taosThreadMutexLock(&pDeleter->mutex);
pOutput->queryEnd = pDeleter->queryEnd; pOutput->queryEnd = pDeleter->queryEnd;
pOutput->useconds = pDeleter->useconds; pOutput->useconds = pDeleter->useconds;
pOutput->precision = pDeleter->pSchema->precision; pOutput->precision = pDeleter->pSchema->precision;
taosThreadMutexUnlock(&pDeleter->mutex); (void)taosThreadMutexUnlock(&pDeleter->mutex);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize); (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize);
taosMemoryFreeClear(pDeleter->nextOutput.pData); taosMemoryFreeClear(pDeleter->nextOutput.pData);
taosArrayDestroy(pDeleter->pParam->pUidList); taosArrayDestroy(pDeleter->pParam->pUidList);
taosMemoryFree(pDeleter->pParam); taosMemoryFree(pDeleter->pParam);
while (!taosQueueEmpty(pDeleter->pDataBlocks)) { while (!taosQueueEmpty(pDeleter->pDataBlocks)) {
SDataDeleterBuf* pBuf = NULL; SDataDeleterBuf* pBuf = NULL;
taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); (void)taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) { if (pBuf != NULL) {
taosMemoryFreeClear(pBuf->pData); taosMemoryFreeClear(pBuf->pData);
@ -228,9 +250,10 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
} }
} }
taosCloseQueue(pDeleter->pDataBlocks); taosCloseQueue(pDeleter->pDataBlocks);
taosThreadMutexDestroy(&pDeleter->mutex); (void)taosThreadMutexDestroy(&pDeleter->mutex);
taosMemoryFree(pDeleter->pManager); taosMemoryFree(pDeleter->pManager);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -247,8 +270,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle));
if (NULL == deleter) { if (NULL == deleter) {
code = terrno;
taosMemoryFree(pParam); taosMemoryFree(pParam);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
@ -276,17 +299,22 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
if (code) { if (code) {
goto _end; goto _end;
} }
taosThreadMutexInit(&deleter->mutex, NULL); code = taosThreadMutexInit(&deleter->mutex, NULL);
if (code) {
goto _end;
}
*pHandle = deleter; *pHandle = deleter;
return code; return code;
_end: _end:
if (deleter != NULL) { if (deleter != NULL) {
destroyDataSinker((SDataSinkHandle*)deleter); (void)destroyDataSinker((SDataSinkHandle*)deleter);
taosMemoryFree(deleter); taosMemoryFree(deleter);
} else { } else {
taosMemoryFree(pManager); taosMemoryFree(pManager);
} }
return code; return code;
} }

View File

@ -63,7 +63,7 @@ typedef struct SDataDispatchHandle {
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is // The length of bitmap is decided by number of rows of this data block, and the length of each column data is
// recorded in the first segment, next to the struct header // recorded in the first segment, next to the struct header
// clang-format on // clang-format on
static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
int32_t numOfCols = 0; int32_t numOfCols = 0;
SNode* pNode; SNode* pNode;
@ -88,6 +88,9 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
if (pHandle->pCompressBuf == NULL) { if (pHandle->pCompressBuf == NULL) {
// allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size
pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8); pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8);
if (NULL == pHandle->pCompressBuf) {
QRY_RET(terrno);
}
pHandle->bufSize = pBuf->allocSize + 8; pHandle->bufSize = pBuf->allocSize + 8;
} else { } else {
if (pHandle->bufSize < pBuf->allocSize + 8) { if (pHandle->bufSize < pBuf->allocSize + 8) {
@ -96,9 +99,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
if (p != NULL) { if (p != NULL) {
pHandle->pCompressBuf = p; pHandle->pCompressBuf = p;
} else { } else {
terrno = TSDB_CODE_OUT_OF_MEMORY; qError("failed to prepare compress buf:%d, code: %x", pHandle->bufSize, terrno);
qError("failed to prepare compress buf:%d, code: out of memory", pHandle->bufSize); return terrno;
return;
} }
} }
} }
@ -114,7 +116,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pEntry->compressed = 0; pEntry->compressed = 0;
pEntry->dataLen = dataLen; pEntry->dataLen = dataLen;
pEntry->rawLen = dataLen; pEntry->rawLen = dataLen;
memcpy(pEntry->data, pHandle->pCompressBuf, dataLen); TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen);
} }
} else { } else {
pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
@ -124,11 +126,13 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
pBuf->useSize += pEntry->dataLen; pBuf->useSize += pEntry->dataLen;
atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); (void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); (void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
return TSDB_CODE_SUCCESS;
} }
static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { static int32_t allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
/* /*
uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery; uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) { if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) {
@ -142,69 +146,73 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput,
pBuf->pData = taosMemoryMalloc(pBuf->allocSize); pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
if (pBuf->pData == NULL) { if (pBuf->pData == NULL) {
qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); qError("SinkNode failed to malloc memory, size:%d, code:%x", pBuf->allocSize, terrno);
return terrno;
} }
return NULL != pBuf->pData; return TSDB_CODE_SUCCESS;
} }
static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { static int32_t updateStatus(SDataDispatchHandle* pDispatcher) {
taosThreadMutexLock(&pDispatcher->mutex); (void)taosThreadMutexLock(&pDispatcher->mutex);
int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks); int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks);
int32_t status = int32_t status =
(0 == blockNums ? DS_BUF_EMPTY (0 == blockNums ? DS_BUF_EMPTY
: (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
pDispatcher->status = status; pDispatcher->status = status;
taosThreadMutexUnlock(&pDispatcher->mutex); (void)taosThreadMutexUnlock(&pDispatcher->mutex);
return status; return status;
} }
static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t getStatus(SDataDispatchHandle* pDispatcher) {
taosThreadMutexLock(&pDispatcher->mutex); (void)taosThreadMutexLock(&pDispatcher->mutex);
int32_t status = pDispatcher->status; int32_t status = pDispatcher->status;
taosThreadMutexUnlock(&pDispatcher->mutex); (void)taosThreadMutexUnlock(&pDispatcher->mutex);
return status; return status;
} }
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
int32_t code = 0; int32_t code = 0;
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
SDataDispatchBuf* pBuf; SDataDispatchBuf* pBuf = NULL;
code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf); code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) { if (code) {
return code; return code;
} }
if (!allocBuf(pDispatcher, pInput, pBuf)) { code = allocBuf(pDispatcher, pInput, pBuf);
if (code) {
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
return TSDB_CODE_OUT_OF_MEMORY;
}
toDataCacheEntry(pDispatcher, pInput, pBuf);
code = taosWriteQitem(pDispatcher->pDataBlocks, pBuf);
if (code != 0) {
return code; return code;
} }
QRY_ERR_JRET(toDataCacheEntry(pDispatcher, pInput, pBuf));
QRY_ERR_JRET(taosWriteQitem(pDispatcher->pDataBlocks, pBuf));
int32_t status = updateStatus(pDispatcher); int32_t status = updateStatus(pDispatcher);
*pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY); *pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return:
taosFreeQitem(pBuf);
return code;
} }
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex); (void)taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = true; pDispatcher->queryEnd = true;
pDispatcher->useconds = useconds; pDispatcher->useconds = useconds;
taosThreadMutexUnlock(&pDispatcher->mutex); (void)taosThreadMutexUnlock(&pDispatcher->mutex);
} }
static void resetDispatcher(struct SDataSinkHandle* pHandle) { static void resetDispatcher(struct SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
taosThreadMutexLock(&pDispatcher->mutex); (void)taosThreadMutexLock(&pDispatcher->mutex);
pDispatcher->queryEnd = false; pDispatcher->queryEnd = false;
taosThreadMutexUnlock(&pDispatcher->mutex); (void)taosThreadMutexUnlock(&pDispatcher->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) {
@ -216,9 +224,9 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow
} }
SDataDispatchBuf* pBuf = NULL; SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); (void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) { if (pBuf != NULL) {
memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); TAOS_MEMCPY(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
} }
@ -243,33 +251,34 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
} }
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfRows = pEntry->numOfRows;
pOutput->numOfCols = pEntry->numOfCols; pOutput->numOfCols = pEntry->numOfCols;
pOutput->compressed = pEntry->compressed; pOutput->compressed = pEntry->compressed;
atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); (void)atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent
pOutput->bufStatus = updateStatus(pDispatcher); pOutput->bufStatus = updateStatus(pDispatcher);
taosThreadMutexLock(&pDispatcher->mutex);
(void)taosThreadMutexLock(&pDispatcher->mutex);
pOutput->queryEnd = pDispatcher->queryEnd; pOutput->queryEnd = pDispatcher->queryEnd;
pOutput->useconds = pDispatcher->useconds; pOutput->useconds = pDispatcher->useconds;
pOutput->precision = pDispatcher->pSchema->precision; pOutput->precision = pDispatcher->pSchema->precision;
taosThreadMutexUnlock(&pDispatcher->mutex); (void)taosThreadMutexUnlock(&pDispatcher->mutex);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
taosMemoryFreeClear(pDispatcher->nextOutput.pData); taosMemoryFreeClear(pDispatcher->nextOutput.pData);
while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { while (!taosQueueEmpty(pDispatcher->pDataBlocks)) {
SDataDispatchBuf* pBuf = NULL; SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); (void)taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
if (pBuf != NULL) { if (pBuf != NULL) {
taosMemoryFreeClear(pBuf->pData); taosMemoryFreeClear(pBuf->pData);
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
@ -280,7 +289,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
taosMemoryFreeClear(pDispatcher->pCompressBuf); taosMemoryFreeClear(pDispatcher->pCompressBuf);
pDispatcher->bufSize = 0; pDispatcher->bufSize = 0;
taosThreadMutexDestroy(&pDispatcher->mutex); (void)taosThreadMutexDestroy(&pDispatcher->mutex);
taosMemoryFree(pDispatcher->pManager); taosMemoryFree(pDispatcher->pManager);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -297,7 +306,6 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle));
if (NULL == dispatcher) { if (NULL == dispatcher) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
@ -318,7 +326,11 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
terrno = code; terrno = code;
goto _return; goto _return;
} }
taosThreadMutexInit(&dispatcher->mutex, NULL); code = taosThreadMutexInit(&dispatcher->mutex, NULL);
if (code) {
terrno = code;
goto _return;
}
if (NULL == dispatcher->pDataBlocks) { if (NULL == dispatcher->pDataBlocks) {
taosMemoryFree(dispatcher); taosMemoryFree(dispatcher);
@ -330,6 +342,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
taosMemoryFree(pManager); taosMemoryFree(pManager);
return terrno; return terrno;
} }

View File

@ -58,10 +58,17 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
SSubmitRspParam* pParam = (SSubmitRspParam*)param; SSubmitRspParam* pParam = (SSubmitRspParam*)param;
SDataInserterHandle* pInserter = pParam->pInserter; SDataInserterHandle* pInserter = pParam->pInserter;
pInserter->submitRes.code = code; if (code) {
pInserter->submitRes.code = code;
}
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2));
if (NULL == pInserter->submitRes.pRsp) {
pInserter->submitRes.code = terrno;
goto _return;
}
SDecoder coder = {0}; SDecoder coder = {0};
tDecoderInit(&coder, pMsg->pData, pMsg->len); tDecoderInit(&coder, pMsg->pData, pMsg->len);
code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp);
@ -77,6 +84,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i);
if (NULL == pRsp) {
pInserter->submitRes.code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _return;
}
if (TSDB_CODE_SUCCESS != pRsp->code) { if (TSDB_CODE_SUCCESS != pRsp->code) {
code = pRsp->code; code = pRsp->code;
taosMemoryFree(pInserter->submitRes.pRsp); taosMemoryFree(pInserter->submitRes.pRsp);
@ -94,8 +105,10 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) {
} }
_return: _return:
tsem_post(&pInserter->ready);
(void)tsem_post(&pInserter->ready);
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -105,11 +118,15 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) { if (NULL == pMsgSendInfo) {
taosMemoryFreeClear(pMsg); taosMemoryFreeClear(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno; return terrno;
} }
SSubmitRspParam* pParam = taosMemoryCalloc(1, sizeof(SSubmitRspParam)); SSubmitRspParam* pParam = taosMemoryCalloc(1, sizeof(SSubmitRspParam));
if (NULL == pParam) {
taosMemoryFreeClear(pMsg);
taosMemoryFreeClear(pMsgSendInfo);
return terrno;
}
pParam->pInserter = pInserter; pParam->pInserter = pInserter;
pMsgSendInfo->param = pParam; pMsgSendInfo->param = pParam;
@ -133,7 +150,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
len += sizeof(SSubmitReq2Msg); len += sizeof(SSubmitReq2Msg);
pBuf = taosMemoryMalloc(len); pBuf = taosMemoryMalloc(len);
if (NULL == pBuf) { if (NULL == pBuf) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId); ((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId);
((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len);
@ -149,6 +166,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
} else { } else {
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
} }
return code; return code;
} }
@ -162,12 +180,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
if (NULL == pReq) { if (NULL == pReq) {
if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
} }
@ -208,6 +224,10 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
} }
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx);
if (NULL == pColInfoData) {
terrno = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
goto _end;
}
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
switch (pColInfoData->info.type) { switch (pColInfoData->info.type) {
@ -217,13 +237,17 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
ASSERT(pColInfoData->info.type == pCol->type); ASSERT(pColInfoData->info.type == pCol->type);
if (colDataIsNull_s(pColInfoData, j)) { if (colDataIsNull_s(pColInfoData, j)) {
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} else { } else {
void* data = colDataGetVarData(pColInfoData, j); void* data = colDataGetVarData(pColInfoData, j);
SValue sv = (SValue){ SValue sv = (SValue){
.type = pCol->type, .nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value .type = pCol->type, .nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} }
break; break;
} }
@ -245,7 +269,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type
taosArrayPush(pVals, &cv); if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} else { } else {
if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && !needSortMerge) { if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && !needSortMerge) {
if (*(int64_t*)var <= lastTs) { if (*(int64_t*)var <= lastTs) {
@ -256,9 +282,11 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
} }
SValue sv = {.type = pCol->type}; SValue sv = {.type = pCol->type};
memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); TAOS_MEMCPY(&sv.val, var, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); if (NULL == taosArrayPush(pVals, &cv)) {
goto _end;
}
} }
} else { } else {
uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
@ -274,7 +302,9 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end; goto _end;
} }
taosArrayPush(tbData.aRowP, &pRow); if (NULL == taosArrayPush(tbData.aRowP, &pRow)) {
goto _end;
}
} }
if (needSortMerge) { if (needSortMerge) {
@ -284,9 +314,12 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
} }
} }
taosArrayPush(pReq->aSubmitTbData, &tbData); if (NULL == taosArrayPush(pReq->aSubmitTbData, &tbData)) {
goto _end;
}
_end: _end:
taosArrayDestroy(pVals); taosArrayDestroy(pVals);
if (terrno != 0) { if (terrno != 0) {
*ppReq = NULL; *ppReq = NULL;
@ -294,9 +327,11 @@ _end:
tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq); taosMemoryFree(pReq);
} }
return terrno; return terrno;
} }
*ppReq = pReq; *ppReq = pReq;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -312,7 +347,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i);
if (NULL == pDataBlock) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid);
if (code) { if (code) {
if (pReq) { if (pReq) {
@ -334,7 +371,9 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
if (!pInserter->explain) { if (!pInserter->explain) {
taosArrayPush(pInserter->pDataBlocks, &pInput->pData); if (NULL == taosArrayPush(pInserter->pDataBlocks, &pInput->pData)) {
return terrno;
}
void* pMsg = NULL; void* pMsg = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen);
@ -350,7 +389,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
return code; return code;
} }
tsem_wait(&pInserter->ready); QRY_ERR_RET(tsem_wait(&pInserter->ready));
if (pInserter->submitRes.code) { if (pInserter->submitRes.code) {
return pInserter->submitRes.code; return pInserter->submitRes.code;
@ -364,10 +403,10 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput,
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
taosThreadMutexLock(&pInserter->mutex); (void)taosThreadMutexLock(&pInserter->mutex);
pInserter->queryEnd = true; pInserter->queryEnd = true;
pInserter->useconds = useconds; pInserter->useconds = useconds;
taosThreadMutexUnlock(&pInserter->mutex); (void)taosThreadMutexUnlock(&pInserter->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
@ -378,12 +417,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRaw
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize); (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize);
taosArrayDestroy(pInserter->pDataBlocks); taosArrayDestroy(pInserter->pDataBlocks);
taosMemoryFree(pInserter->pSchema); taosMemoryFree(pInserter->pSchema);
taosMemoryFree(pInserter->pParam); taosMemoryFree(pInserter->pParam);
taosHashCleanup(pInserter->pCols); taosHashCleanup(pInserter->pCols);
taosThreadMutexDestroy(&pInserter->mutex); (void)taosThreadMutexDestroy(&pInserter->mutex);
taosMemoryFree(pInserter->pManager); taosMemoryFree(pInserter->pManager);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -401,7 +440,6 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle));
if (NULL == inserter) { if (NULL == inserter) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
@ -432,28 +470,31 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
} }
inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES); inserter->pDataBlocks = taosArrayInit(1, POINTER_BYTES);
taosThreadMutexInit(&inserter->mutex, NULL);
if (NULL == inserter->pDataBlocks) { if (NULL == inserter->pDataBlocks) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
QRY_ERR_JRET(taosThreadMutexInit(&inserter->mutex, NULL));
inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols; inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols;
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT),
false, HASH_NO_LOCK); false, HASH_NO_LOCK);
if (NULL == inserter->pCols) {
goto _return;
}
SNode* pNode = NULL; SNode* pNode = NULL;
int32_t i = 0; int32_t i = 0;
FOREACH(pNode, pInserterNode->pCols) { FOREACH(pNode, pInserterNode->pCols) {
SColumnNode* pCol = (SColumnNode*)pNode; SColumnNode* pCol = (SColumnNode*)pNode;
taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)); QRY_ERR_JRET(taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId)));
if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) { if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) {
inserter->fullOrderColList = false; inserter->fullOrderColList = false;
} }
++i; ++i;
} }
tsem_init(&inserter->ready, 0, 0); QRY_ERR_JRET(tsem_init(&inserter->ready, 0, 0));
*pHandle = inserter; *pHandle = inserter;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -461,7 +502,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
_return: _return:
if (inserter) { if (inserter) {
destroyDataSinker((SDataSinkHandle*)inserter); (void)destroyDataSinker((SDataSinkHandle*)inserter);
taosMemoryFree(inserter); taosMemoryFree(inserter);
} else { } else {
taosMemoryFree(pManager); taosMemoryFree(pManager);

View File

@ -23,20 +23,20 @@ SDataSinkStat gDataSinkStat = {0};
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) { int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) {
SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager)); SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager));
if (NULL == pSinkManager) { if (NULL == pSinkManager) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pSinkManager->cfg = *cfg; pSinkManager->cfg = *cfg;
pSinkManager->pAPI = pAPI; pSinkManager->pAPI = pAPI;
*ppSinkManager = pSinkManager; *ppSinkManager = pSinkManager;
return 0; // to avoid compiler eror return TSDB_CODE_SUCCESS; // to avoid compiler eror
} }
int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) { int32_t dsDataSinkGetCacheSize(SDataSinkStat* pStat) {
pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize); pStat->cachedSize = atomic_load_64(&gDataSinkStat.cachedSize);
return 0; return TSDB_CODE_SUCCESS;
} }
int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) { int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void* pParam, const char* id) {
@ -56,6 +56,7 @@ int32_t dsCreateDataSinker(void* pSinkManager, const SDataSinkNode* pDataSink, D
taosMemoryFree(pSinkManager); taosMemoryFree(pSinkManager);
qError("invalid input node type:%d, %s", nodeType(pDataSink), id); qError("invalid input node type:%d, %s", nodeType(pDataSink), id);
return TSDB_CODE_QRY_INVALID_INPUT; return TSDB_CODE_QRY_INVALID_INPUT;
} }
@ -97,6 +98,6 @@ void dsScheduleProcess(void* ahandle, void* pItem) {
void dsDestroyDataSinker(DataSinkHandle handle) { void dsDestroyDataSinker(DataSinkHandle handle) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
pHandleImpl->fDestroy(pHandleImpl); (void)pHandleImpl->fDestroy(pHandleImpl);
taosMemoryFree(pHandleImpl); taosMemoryFree(pHandleImpl);
} }

View File

@ -105,7 +105,7 @@ static FORCE_INLINE bool tableNeedCache(int64_t uid, SStbJoinPrevJoinCtx* pPrev,
return (NULL == num) ? false : true; return (NULL == num) ? false : true;
} }
static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) { static int32_t updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev; SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post; SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
SStbJoinTableList* pNode = pPrev->pListHead; SStbJoinTableList* pNode = pPrev->pListHead;
@ -141,32 +141,38 @@ static void updatePostJoinCurrTableInfo(SStbJoinDynCtrlInfo* pStbJoin)
pPost->leftNeedCache = tableNeedCache(*leftUid, pPrev, pPost, false, pStbJoin->basic.batchFetch); pPost->leftNeedCache = tableNeedCache(*leftUid, pPrev, pPost, false, pStbJoin->basic.batchFetch);
pPost->rightNeedCache = tableNeedCache(*rightUid, pPrev, pPost, true, pStbJoin->basic.batchFetch); pPost->rightNeedCache = tableNeedCache(*rightUid, pPrev, pPost, true, pStbJoin->basic.batchFetch);
if (pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) { if (!pStbJoin->basic.batchFetch && pPost->rightNeedCache && rightPrevUid != pPost->rightCurrUid) {
tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0); QRY_ERR_RET(tSimpleHashPut(pPrev->rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid), NULL, 0));
pStbJoin->execInfo.rightCacheNum++; pStbJoin->execInfo.rightCacheNum++;
} }
return TSDB_CODE_SUCCESS;
} }
static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) { static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid, bool needCache, SOperatorParam* pChild) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); *ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) { if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM); freeOperatorParam(pChild, OP_GET_PARAM);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
if (pChild) { if (pChild) {
(*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES); (*ppRes)->pChildren = taosArrayInit(1, POINTER_BYTES);
if (NULL == (*ppRes)->pChildren) { if (NULL == (*ppRes)->pChildren) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM); freeOperatorParam(pChild, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) { if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild)) {
code = terrno;
freeOperatorParam(pChild, OP_GET_PARAM); freeOperatorParam(pChild, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
} else { } else {
(*ppRes)->pChildren = NULL; (*ppRes)->pChildren = NULL;
@ -174,9 +180,10 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down
SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam)); SGcOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcOperatorParam));
if (NULL == pGc) { if (NULL == pGc) {
code = terrno;
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
pGc->sessionId = atomic_add_fetch_64(&gSessionId, 1); pGc->sessionId = atomic_add_fetch_64(&gSessionId, 1);
@ -194,16 +201,18 @@ static int32_t buildGroupCacheOperatorParam(SOperatorParam** ppRes, int32_t down
static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid) { static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t vgId, int64_t tbUid) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); *ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) { if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
(*ppRes)->pChildren = NULL; (*ppRes)->pChildren = NULL;
SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam)); SGcNotifyOperatorParam* pGc = taosMemoryMalloc(sizeof(SGcNotifyOperatorParam));
if (NULL == pGc) { if (NULL == pGc) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
pGc->downstreamIdx = downstreamIdx; pGc->downstreamIdx = downstreamIdx;
@ -221,13 +230,13 @@ static int32_t buildGroupCacheNotifyOperatorParam(SOperatorParam** ppRes, int32_
static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) { static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); *ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) { if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
(*ppRes)->pChildren = NULL; (*ppRes)->pChildren = NULL;
SExchangeOperatorParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorParam)); SExchangeOperatorParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorParam));
if (NULL == pExc) { if (NULL == pExc) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pExc->multiParams = false; pExc->multiParams = false;
@ -237,7 +246,7 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst
pExc->basic.uidList = taosArrayInit(1, sizeof(int64_t)); pExc->basic.uidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pExc->basic.uidList) { if (NULL == pExc->basic.uidList) {
taosMemoryFree(pExc); taosMemoryFree(pExc);
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
taosArrayPush(pExc->basic.uidList, pUid); taosArrayPush(pExc->basic.uidList, pUid);
@ -252,14 +261,14 @@ static int32_t buildExchangeOperatorParam(SOperatorParam** ppRes, int32_t downst
static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) { static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, SSHashObj* pVg) {
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); *ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) { if (NULL == *ppRes) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
(*ppRes)->pChildren = NULL; (*ppRes)->pChildren = NULL;
SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam)); SExchangeOperatorBatchParam* pExc = taosMemoryMalloc(sizeof(SExchangeOperatorBatchParam));
if (NULL == pExc) { if (NULL == pExc) {
taosMemoryFreeClear(*ppRes); taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pExc->multiParams = true; pExc->multiParams = true;
@ -267,7 +276,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
if (NULL == pExc->pBatchs) { if (NULL == pExc->pBatchs) {
taosMemoryFree(pExc); taosMemoryFree(pExc);
taosMemoryFreeClear(*ppRes); taosMemoryFreeClear(*ppRes);
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam); tSimpleHashSetFreeFp(pExc->pBatchs, freeExchangeGetBasicOperatorParam);
@ -283,7 +292,7 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
basic.uidList = pUidList; basic.uidList = pUidList;
basic.tableSeq = false; basic.tableSeq = false;
tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic)); QRY_ERR_RET(tSimpleHashPut(pExc->pBatchs, pVgId, sizeof(*pVgId), &basic, sizeof(basic)));
qTrace("build downstreamIdx %d batch scan, vgId:%d, uidNum:%" PRId64, downstreamIdx, *pVgId, (int64_t)taosArrayGetSize(pUidList)); qTrace("build downstreamIdx %d batch scan, vgId:%d, uidNum:%" PRId64, downstreamIdx, *pVgId, (int64_t)taosArrayGetSize(pUidList));
*(SArray**)p = NULL; *(SArray**)p = NULL;
@ -298,39 +307,45 @@ static int32_t buildBatchExchangeOperatorParam(SOperatorParam** ppRes, int32_t d
static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) { static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initParam, SOperatorParam* pChild0, SOperatorParam* pChild1) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); *ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) { if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) { if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) { if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
code = terrno;
freeOperatorParam(pChild0, OP_GET_PARAM); freeOperatorParam(pChild0, OP_GET_PARAM);
freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) { if (NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
code = terrno;
freeOperatorParam(pChild1, OP_GET_PARAM); freeOperatorParam(pChild1, OP_GET_PARAM);
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam)); SSortMergeJoinOperatorParam* pJoin = taosMemoryMalloc(sizeof(SSortMergeJoinOperatorParam));
if (NULL == pJoin) { if (NULL == pJoin) {
code = terrno;
freeOperatorParam(*ppRes, OP_GET_PARAM); freeOperatorParam(*ppRes, OP_GET_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
pJoin->initDownstream = initParam; pJoin->initDownstream = initParam;
@ -343,31 +358,36 @@ static int32_t buildMergeJoinOperatorParam(SOperatorParam** ppRes, bool initPara
static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) { static int32_t buildMergeJoinNotifyOperatorParam(SOperatorParam** ppRes, SOperatorParam* pChild0, SOperatorParam* pChild1) {
int32_t code = TSDB_CODE_SUCCESS;
*ppRes = taosMemoryMalloc(sizeof(SOperatorParam)); *ppRes = taosMemoryMalloc(sizeof(SOperatorParam));
if (NULL == *ppRes) { if (NULL == *ppRes) {
code = terrno;
freeOperatorParam(pChild0, OP_NOTIFY_PARAM); freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
(*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES); (*ppRes)->pChildren = taosArrayInit(2, POINTER_BYTES);
if (NULL == *ppRes) { if (NULL == *ppRes) {
code = terrno;
taosMemoryFreeClear(*ppRes); taosMemoryFreeClear(*ppRes);
freeOperatorParam(pChild0, OP_NOTIFY_PARAM); freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) { if (pChild0 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild0)) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
freeOperatorParam(pChild0, OP_NOTIFY_PARAM); freeOperatorParam(pChild0, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) { if (pChild1 && NULL == taosArrayPush((*ppRes)->pChildren, &pChild1)) {
code = terrno;
freeOperatorParam(*ppRes, OP_NOTIFY_PARAM); freeOperatorParam(*ppRes, OP_NOTIFY_PARAM);
freeOperatorParam(pChild1, OP_NOTIFY_PARAM); freeOperatorParam(pChild1, OP_NOTIFY_PARAM);
*ppRes = NULL; *ppRes = NULL;
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
(*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN; (*ppRes)->opType = QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN;
@ -407,9 +427,11 @@ static int32_t buildBatchTableScanOperatorParam(SOperatorParam** ppRes, int32_t
static int32_t buildSingleTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) { static int32_t buildSingleTableScanOperatorParam(SOperatorParam** ppRes, int32_t downstreamIdx, int32_t* pVgId, int64_t* pUid) {
SArray* pUidList = taosArrayInit(1, sizeof(int64_t)); SArray* pUidList = taosArrayInit(1, sizeof(int64_t));
if (NULL == pUidList) { if (NULL == pUidList) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
}
if (NULL == taosArrayPush(pUidList, pUid)) {
return terrno;
} }
taosArrayPush(pUidList, pUid);
int32_t code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true); int32_t code = buildTableScanOperatorParam(ppRes, pUidList, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, true);
taosArrayDestroy(pUidList); taosArrayDestroy(pUidList);
@ -435,7 +457,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64, qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid); rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid);
updatePostJoinCurrTableInfo(&pInfo->stbJoin); QRY_ERR_RET(updatePostJoinCurrTableInfo(&pInfo->stbJoin));
if (pInfo->stbJoin.basic.batchFetch) { if (pInfo->stbJoin.basic.batchFetch) {
if (pPrev->leftHash) { if (pPrev->leftHash) {
@ -538,53 +560,56 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost
return optrDefaultNotifyFn(pOperator->pDownstream[1], pMergeJoinParam); return optrDefaultNotifyFn(pOperator->pDownstream[1], pMergeJoinParam);
} }
static void handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) { static int32_t handleSeqJoinCurrRetrieveEnd(SOperatorInfo* pOperator, SStbJoinDynCtrlInfo* pStbJoin) {
SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post; SStbJoinPostJoinCtx* pPost = &pStbJoin->ctx.post;
pPost->isStarted = false; pPost->isStarted = false;
if (pStbJoin->basic.batchFetch) { if (pStbJoin->basic.batchFetch) {
return; return TSDB_CODE_SUCCESS;
} }
if (pPost->leftNeedCache) { if (pPost->leftNeedCache) {
uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); uint32_t* num = tSimpleHashGet(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
if (num && --(*num) <= 0) { if (num && --(*num) <= 0) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid)); (void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, &pPost->leftCurrUid, sizeof(pPost->leftCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, true); QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, true));
} }
} }
if (!pPost->rightNeedCache) { if (!pPost->rightNeedCache) {
void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); void* v = tSimpleHashGet(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
if (NULL != v) { if (NULL != v) {
tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid)); (void)tSimpleHashRemove(pStbJoin->ctx.prev.rightCache, &pPost->rightCurrUid, sizeof(pPost->rightCurrUid));
notifySeqJoinTableCacheEnd(pOperator, pPost, false); QRY_ERR_RET(notifySeqJoinTableCacheEnd(pOperator, pPost, false));
} }
} }
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE void seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) { static FORCE_INLINE int32_t seqJoinContinueCurrRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinPostJoinCtx* pPost = &pInfo->stbJoin.ctx.post; SStbJoinPostJoinCtx* pPost = &pInfo->stbJoin.ctx.post;
SStbJoinPrevJoinCtx* pPrev = &pInfo->stbJoin.ctx.prev; SStbJoinPrevJoinCtx* pPrev = &pInfo->stbJoin.ctx.prev;
if (!pPost->isStarted) { if (!pPost->isStarted) {
return; return TSDB_CODE_SUCCESS;
} }
qDebug("%s dynQueryCtrl continue to retrieve block from post op", GET_TASKID(pOperator->pTaskInfo)); qDebug("%s dynQueryCtrl continue to retrieve block from post op", GET_TASKID(pOperator->pTaskInfo));
*ppRes = getNextBlockFromDownstream(pOperator, 1); *ppRes = getNextBlockFromDownstream(pOperator, 1);
if (NULL == *ppRes) { if (NULL == *ppRes) {
handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin); QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, &pInfo->stbJoin));
pPrev->pListHead->readIdx++; pPrev->pListHead->readIdx++;
} else { } else {
pInfo->stbJoin.execInfo.postBlkNum++; pInfo->stbJoin.execInfo.postBlkNum++;
pInfo->stbJoin.execInfo.postBlkRows += (*ppRes)->info.rows; pInfo->stbJoin.execInfo.postBlkRows += (*ppRes)->info.rows;
return;
} }
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, int32_t keySize, void* pVal, int32_t valSize) { static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, int32_t keySize, void* pVal, int32_t valSize) {
@ -592,35 +617,38 @@ static FORCE_INLINE int32_t addToJoinVgroupHash(SSHashObj* pHash, void* pKey, in
if (NULL == ppArray) { if (NULL == ppArray) {
SArray* pArray = taosArrayInit(10, valSize); SArray* pArray = taosArrayInit(10, valSize);
if (NULL == pArray) { if (NULL == pArray) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
if (NULL == taosArrayPush(pArray, pVal)) { if (NULL == taosArrayPush(pArray, pVal)) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
if (tSimpleHashPut(pHash, pKey, keySize, &pArray, POINTER_BYTES)) { if (tSimpleHashPut(pHash, pKey, keySize, &pArray, POINTER_BYTES)) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (NULL == taosArrayPush(*ppArray, pVal)) { if (NULL == taosArrayPush(*ppArray, pVal)) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnceHash, void* pKey, int32_t keySize) { static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnceHash, void* pKey, int32_t keySize) {
int32_t code = TSDB_CODE_SUCCESS;
uint32_t* pNum = tSimpleHashGet(pHash, pKey, keySize); uint32_t* pNum = tSimpleHashGet(pHash, pKey, keySize);
if (NULL == pNum) { if (NULL == pNum) {
uint32_t n = 1; uint32_t n = 1;
if (tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n))) { code = tSimpleHashPut(pHash, pKey, keySize, &n, sizeof(n));
return TSDB_CODE_OUT_OF_MEMORY; if (code) {
return code;
} }
if (tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0)) { code = tSimpleHashPut(pOnceHash, pKey, keySize, NULL, 0);
return TSDB_CODE_OUT_OF_MEMORY; if (code) {
return code;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -633,7 +661,7 @@ static FORCE_INLINE int32_t addToJoinTableHash(SSHashObj* pHash, SSHashObj* pOnc
break; break;
default: default:
if (1 == (*pNum)) { if (1 == (*pNum)) {
tSimpleHashRemove(pOnceHash, pKey, keySize); (void)tSimpleHashRemove(pOnceHash, pKey, keySize);
} }
(*pNum)++; (*pNum)++;
break; break;
@ -655,23 +683,40 @@ static void freeStbJoinTableList(SStbJoinTableList* pList) {
} }
static int32_t appendStbJoinTableList(SStbJoinPrevJoinCtx* pCtx, int64_t rows, int32_t* pLeftVg, int64_t* pLeftUid, int32_t* pRightVg, int64_t* pRightUid) { static int32_t appendStbJoinTableList(SStbJoinPrevJoinCtx* pCtx, int64_t rows, int32_t* pLeftVg, int64_t* pLeftUid, int32_t* pRightVg, int64_t* pRightUid) {
int32_t code = TSDB_CODE_SUCCESS;
SStbJoinTableList* pNew = taosMemoryMalloc(sizeof(SStbJoinTableList)); SStbJoinTableList* pNew = taosMemoryMalloc(sizeof(SStbJoinTableList));
if (NULL == pNew) { if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pNew->pLeftVg = taosMemoryMalloc(rows * sizeof(*pLeftVg)); pNew->pLeftVg = taosMemoryMalloc(rows * sizeof(*pLeftVg));
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid)); if (NULL == pNew->pLeftVg) {
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg)); code = terrno;
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pLeftVg || NULL == pNew->pLeftUid || NULL == pNew->pRightVg || NULL == pNew->pRightUid) {
freeStbJoinTableList(pNew); freeStbJoinTableList(pNew);
return TSDB_CODE_OUT_OF_MEMORY; return code;
}
pNew->pLeftUid = taosMemoryMalloc(rows * sizeof(*pLeftUid));
if (NULL == pNew->pLeftUid) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
pNew->pRightVg = taosMemoryMalloc(rows * sizeof(*pRightVg));
if (NULL == pNew->pRightVg) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
}
pNew->pRightUid = taosMemoryMalloc(rows * sizeof(*pRightUid));
if (NULL == pNew->pRightUid) {
code = terrno;
freeStbJoinTableList(pNew);
return code;
} }
memcpy(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg)); TAOS_MEMCPY(pNew->pLeftVg, pLeftVg, rows * sizeof(*pLeftVg));
memcpy(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid)); TAOS_MEMCPY(pNew->pLeftUid, pLeftUid, rows * sizeof(*pLeftUid));
memcpy(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg)); TAOS_MEMCPY(pNew->pRightVg, pRightVg, rows * sizeof(*pRightVg));
memcpy(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid)); TAOS_MEMCPY(pNew->pRightUid, pRightUid, rows * sizeof(*pRightUid));
pNew->readIdx = 0; pNew->readIdx = 0;
pNew->uidNum = rows; pNew->uidNum = rows;
@ -693,9 +738,21 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin; SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SColumnInfoData* pVg0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[0]); SColumnInfoData* pVg0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[0]);
if (NULL == pVg0) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pVg1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[1]); SColumnInfoData* pVg1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.vgSlot[1]);
if (NULL == pVg1) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pUid0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[0]); SColumnInfoData* pUid0 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[0]);
if (NULL == pUid0) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pUid1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[1]); SColumnInfoData* pUid1 = taosArrayGet(pBlock->pDataBlock, pStbJoin->basic.uidSlot[1]);
if (NULL == pUid1) {
QRY_ERR_JRET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pStbJoin->basic.batchFetch) { if (pStbJoin->basic.batchFetch) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) { for (int32_t i = 0; i < pBlock->info.rows; ++i) {
@ -731,6 +788,8 @@ static void doBuildStbJoinTableHash(SOperatorInfo* pOperator, SSDataBlock* pBloc
} }
} }
_return:
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
pOperator->pTaskInfo->code = code; pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code); T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
@ -754,7 +813,7 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
uint64_t* pUid = NULL; uint64_t* pUid = NULL;
int32_t iter = 0; int32_t iter = 0;
while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) { while (NULL != (pUid = tSimpleHashIterate(pStbJoin->ctx.prev.onceTable, pUid, &iter))) {
tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid)); (void)tSimpleHashRemove(pStbJoin->ctx.prev.leftCache, pUid, sizeof(*pUid));
} }
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache); pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
@ -789,7 +848,7 @@ static void buildStbJoinTableList(SOperatorInfo* pOperator) {
pStbJoin->ctx.prev.joinBuild = true; pStbJoin->ctx.prev.joinBuild = true;
} }
static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) { static int32_t seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin; SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev; SStbJoinPrevJoinCtx* pPrev = &pStbJoin->ctx.prev;
@ -805,17 +864,17 @@ static void seqJoinLaunchNewRetrieve(SOperatorInfo* pOperator, SSDataBlock** ppR
seqJoinLaunchNewRetrieveImpl(pOperator, ppRes); seqJoinLaunchNewRetrieveImpl(pOperator, ppRes);
if (*ppRes) { if (*ppRes) {
return; return TSDB_CODE_SUCCESS;
} }
handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin); QRY_ERR_RET(handleSeqJoinCurrRetrieveEnd(pOperator, pStbJoin));
pPrev->pListHead->readIdx++; pPrev->pListHead->readIdx++;
} }
*ppRes = NULL; *ppRes = NULL;
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
return; return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* pStbJoin, SSDataBlock* pBlock) { static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo* pStbJoin, SSDataBlock* pBlock) {
@ -825,6 +884,7 @@ static FORCE_INLINE SSDataBlock* seqStableJoinComposeRes(SStbJoinDynCtrlInfo*
SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) { SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
SDynQueryCtrlOperatorInfo* pInfo = pOperator->info; SDynQueryCtrlOperatorInfo* pInfo = pOperator->info;
SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin; SStbJoinDynCtrlInfo* pStbJoin = (SStbJoinDynCtrlInfo*)&pInfo->stbJoin;
SSDataBlock* pRes = NULL; SSDataBlock* pRes = NULL;
@ -846,18 +906,23 @@ SSDataBlock* seqStableJoin(SOperatorInfo* pOperator) {
} }
} }
seqJoinContinueCurrRetrieve(pOperator, &pRes); QRY_ERR_JRET(seqJoinContinueCurrRetrieve(pOperator, &pRes));
if (pRes) { if (pRes) {
goto _return; goto _return;
} }
seqJoinLaunchNewRetrieve(pOperator, &pRes); QRY_ERR_JRET(seqJoinLaunchNewRetrieve(pOperator, &pRes));
_return: _return:
if (pOperator->cost.openCost == 0) { if (pOperator->cost.openCost == 0) {
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
} }
if (code) {
pOperator->pTaskInfo->code = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pOperator->pTaskInfo->code);
}
return pRes ? seqStableJoinComposeRes(pStbJoin, pRes) : NULL; return pRes ? seqStableJoinComposeRes(pStbJoin, pRes) : NULL;
} }
@ -866,24 +931,24 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
if (batchFetch) { if (batchFetch) {
pPrev->leftHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); pPrev->leftHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->leftHash) { if (NULL == pPrev->leftHash) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pPrev->rightHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT)); pPrev->rightHash = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
if (NULL == pPrev->rightHash) { if (NULL == pPrev->rightHash) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
} else { } else {
pPrev->leftCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); pPrev->leftCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->leftCache) { if (NULL == pPrev->leftCache) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pPrev->rightCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); pPrev->rightCache = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->rightCache) { if (NULL == pPrev->rightCache) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pPrev->onceTable = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); pPrev->onceTable = tSimpleHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (NULL == pPrev->onceTable) { if (NULL == pPrev->onceTable) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
} }
@ -893,13 +958,16 @@ int32_t initSeqStbJoinTableHash(SStbJoinPrevJoinCtx* pPrev, bool batchFetch) {
SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo) { SDynQueryCtrlPhysiNode* pPhyciNode, SExecTaskInfo* pTaskInfo) {
SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
__optr_fn_t nextFp = NULL;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) { __optr_fn_t nextFp = NULL;
code = TSDB_CODE_OUT_OF_MEMORY; SDynQueryCtrlOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SDynQueryCtrlOperatorInfo));
if (pInfo == NULL) {
code = terrno;
goto _error;
}
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL) {
code = terrno;
goto _error; goto _error;
} }
@ -913,7 +981,7 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32
pInfo->qType = pPhyciNode->qType; pInfo->qType = pPhyciNode->qType;
switch (pInfo->qType) { switch (pInfo->qType) {
case DYN_QTYPE_STB_HASH: case DYN_QTYPE_STB_HASH:
memcpy(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin)); TAOS_MEMCPY(&pInfo->stbJoin.basic, &pPhyciNode->stbJoin, sizeof(pPhyciNode->stbJoin));
pInfo->stbJoin.outputBlkId = pPhyciNode->node.pOutputDataBlockDesc->dataBlockId; pInfo->stbJoin.outputBlkId = pPhyciNode->node.pOutputDataBlockDesc->dataBlockId;
code = initSeqStbJoinTableHash(&pInfo->stbJoin.ctx.prev, pInfo->stbJoin.basic.batchFetch); code = initSeqStbJoinTableHash(&pInfo->stbJoin.ctx.prev, pInfo->stbJoin.basic.batchFetch);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -934,12 +1002,14 @@ SOperatorInfo* createDynQueryCtrlOperatorInfo(SOperatorInfo** pDownstream, int32
return pOperator; return pOperator;
_error: _error:
if (pInfo != NULL) { if (pInfo != NULL) {
destroyDynQueryCtrlOperator(pInfo); destroyDynQueryCtrlOperator(pInfo);
} }
taosMemoryFree(pOperator); taosMemoryFree(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
return NULL; return NULL;
} }

View File

@ -1627,7 +1627,8 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)
|| (strcmp(pName, "_group_const_value") == 0)) {
pValCtx[num++] = &pCtx[i]; pValCtx[num++] = &pCtx[i];
} else if (fmIsSelectFunc(pCtx[i].functionId)) { } else if (fmIsSelectFunc(pCtx[i].functionId)) {
void* data = taosHashGet(pSelectFuncs, pName, strlen(pName)); void* data = taosHashGet(pSelectFuncs, pName, strlen(pName));
@ -1683,14 +1684,17 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) { if (fmIsAggFunc(pCtx->functionId) || fmIsIndefiniteRowsFunc(pCtx->functionId)) {
bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId); bool isUdaf = fmIsUserDefinedFunc(pCtx->functionId);
if (!isUdaf) { if (!isUdaf) {
// TODO(xxx) : need handle return value of fmGetFuncExecFuncs.
fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
} else { } else {
char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; char* udfName = pExpr->pExpr->_function.pFunctNode->functionName;
pCtx->udfName = taosStrdup(udfName); pCtx->udfName = taosStrdup(udfName);
// TODO(xxx) : need handle return value of fmGetUdafExecFuncs.
fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
} }
pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
} else { } else {
// TODO(xxx) : need handle return value of fmGetScalarFuncExecFuncs.
fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp); fmGetScalarFuncExecFuncs(pCtx->functionId, &pCtx->sfp);
if (pCtx->sfp.getEnv != NULL) { if (pCtx->sfp.getEnv != NULL) {
pCtx->sfp.getEnv(pExpr->pExpr->_function.pFunctNode, &env); pCtx->sfp.getEnv(pExpr->pExpr->_function.pFunctNode, &env);

View File

@ -521,8 +521,8 @@ int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t n
if (!pResInfo->initialized) { if (!pResInfo->initialized) {
if (pCtx[i].functionId != -1) { if (pCtx[i].functionId != -1) {
bool ini = pCtx[i].fpSet.init(&pCtx[i], pResInfo); int32_t code = pCtx[i].fpSet.init(&pCtx[i], pResInfo);
if (!ini && fmIsUserDefinedFunc(pCtx[i].functionId)) { if (code != TSDB_CODE_SUCCESS && fmIsUserDefinedFunc(pCtx[i].functionId)){
pResInfo->initialized = false; pResInfo->initialized = false;
return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; return TSDB_CODE_UDF_FUNC_EXEC_FAILURE;
} }
@ -638,7 +638,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR
pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
if (pCtx[j].fpSet.finalize) { if (pCtx[j].fpSet.finalize) {
if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0) { if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_key") == 0 ||
strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_group_const_value") == 0) {
// for groupkey along with functions that output multiple lines(e.g. Histogram) // for groupkey along with functions that output multiple lines(e.g. Histogram)
// need to match groupkey result for each output row of that function. // need to match groupkey result for each output row of that function.
if (pCtx[j].resultInfo->numOfRes != 0) { if (pCtx[j].resultInfo->numOfRes != 0) {

View File

@ -30,9 +30,9 @@
static void removeGroupCacheFile(SGroupCacheFileInfo* pFileInfo) { static void removeGroupCacheFile(SGroupCacheFileInfo* pFileInfo) {
if (pFileInfo->fd.fd) { if (pFileInfo->fd.fd) {
taosCloseFile(&pFileInfo->fd.fd); (void)taosCloseFile(&pFileInfo->fd.fd);
pFileInfo->fd.fd = NULL; pFileInfo->fd.fd = NULL;
taosThreadMutexDestroy(&pFileInfo->fd.mutex); (void)taosThreadMutexDestroy(&pFileInfo->fd.mutex);
} }
pFileInfo->deleted = true; pFileInfo->deleted = true;
} }
@ -88,7 +88,7 @@ static void logGroupCacheExecInfo(SGroupCacheOperatorInfo* pGrpCacheOperator) {
static void freeSGcSessionCtx(void* p) { static void freeSGcSessionCtx(void* p) {
SGcSessionCtx* pSession = p; SGcSessionCtx* pSession = p;
if (pSession->semInit) { if (pSession->semInit) {
tsem_destroy(&pSession->waitSem); (void)tsem_destroy(&pSession->waitSem);
} }
} }
@ -148,6 +148,10 @@ void blockDataDeepCleanup(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (NULL == p) {
qError("fail to get %dth col in dataBlock, numOfCols:%d", i, (int32_t)numOfCols);
continue;
}
taosMemoryFreeClear(p->pData); taosMemoryFreeClear(p->pData);
if (IS_VAR_DATA_TYPE(p->info.type)) { if (IS_VAR_DATA_TYPE(p->info.type)) {
taosMemoryFreeClear(p->varmeta.offset); taosMemoryFreeClear(p->varmeta.offset);
@ -196,10 +200,14 @@ static FORCE_INLINE int32_t initOpenCacheFile(SGroupCacheFileFd* pFileFd, char*
TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE|TD_FILE_AUTO_DEL); TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE|TD_FILE_AUTO_DEL);
//TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE); //TdFilePtr newFd = taosOpenFile(filename, TD_FILE_CREATE|TD_FILE_READ|TD_FILE_WRITE);
if (NULL == newFd) { if (NULL == newFd) {
return TAOS_SYSTEM_ERROR(errno); QRY_ERR_RET(TAOS_SYSTEM_ERROR(errno));
} }
pFileFd->fd = newFd; pFileFd->fd = newFd;
taosThreadMutexInit(&pFileFd->mutex, NULL); int32_t code = taosThreadMutexInit(&pFileFd->mutex, NULL);
if (code) {
qError("taosThreadMutexInit failed, code:%x", code);
QRY_ERR_RET(code);
}
qTrace("file path %s created", filename); qTrace("file path %s created", filename);
@ -218,13 +226,16 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S
SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) { if (NULL == pTmp) {
sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId); (void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%d", fileId);
SGroupCacheFileInfo newFile = {0}; SGroupCacheFileInfo newFile = {0};
taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)); if (taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile))) {
QRY_ERR_RET(terrno);
}
pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) { if (NULL == pTmp) {
return TSDB_CODE_OUT_OF_MEMORY; qError("fail to get file %d from pCacheFile", fileId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
} }
} }
@ -240,14 +251,14 @@ static int32_t acquireFdFromFileCtx(SGcFileCacheCtx* pFileCtx, int32_t fileId, S
} }
} }
taosThreadMutexLock(&pTmp->fd.mutex); (void)taosThreadMutexLock(&pTmp->fd.mutex);
*ppFd = &pTmp->fd; *ppFd = &pTmp->fd;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE void releaseFdToFileCtx(SGroupCacheFileFd* pFd) { static FORCE_INLINE void releaseFdToFileCtx(SGroupCacheFileFd* pFd) {
taosThreadMutexUnlock(&pFd->mutex); (void)taosThreadMutexUnlock(&pFd->mutex);
} }
static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamCtx* pCtx, SGcBlkBufInfo* pHead) { static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamCtx* pCtx, SGcBlkBufInfo* pHead) {
@ -275,7 +286,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId; int64_t blkId = pHead->basic.blkId;
pHead = pHead->next; pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
continue; continue;
} }
@ -295,7 +306,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId; int64_t blkId = pHead->basic.blkId;
pHead = pHead->next; pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
continue; continue;
} }
@ -321,7 +332,7 @@ static int32_t saveBlocksToDisk(SGroupCacheOperatorInfo* pGCache, SGcDownstreamC
int64_t blkId = pHead->basic.blkId; int64_t blkId = pHead->basic.blkId;
pHead = pHead->next; pHead = pHead->next;
taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId)); (void)taosHashRemove(pGCache->blkCache.pDirtyBlk, &blkId, sizeof(blkId));
} }
_return: _return:
@ -330,7 +341,7 @@ _return:
taosHashRelease(pGrpHash, pGroup); taosHashRelease(pGrpHash, pGroup);
} }
atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1); (void)atomic_val_compare_exchange_32(&pGCache->blkCache.writeDownstreamId, pCtx->id, -1);
return code; return code;
} }
@ -341,7 +352,8 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr
} }
pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId)); pBufInfo = taosHashGet(pCache->pDirtyBlk, &pBufInfo->basic.blkId, sizeof(pBufInfo->basic.blkId));
if (NULL == pBufInfo) { if (NULL == pBufInfo) {
return TSDB_CODE_OUT_OF_MEMORY; qError("fail to get blk %" PRId64 " from pCache->pDirtyBlk", pBufInfo->basic.blkId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
} }
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SGcBlkBufInfo* pWriteHead = NULL; SGcBlkBufInfo* pWriteHead = NULL;
@ -424,7 +436,7 @@ static int32_t addBlkToBufCache(struct SOperatorInfo* pOperator, SSDataBlock* pB
qError("group cache add block to cache failed, size:%" PRId64, bufSize); qError("group cache add block to cache failed, size:%" PRId64, bufSize);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
blockDataToBuf(pBufInfo->pBuf, pBlock); QRY_ERR_RET(blockDataToBuf(pBufInfo->pBuf, pBlock));
SGcFileCacheCtx* pFileCtx = pGCache->batchFetch ? &pCtx->fileCtx : &pGroup->pVgCtx->fileCtx; SGcFileCacheCtx* pFileCtx = pGCache->batchFetch ? &pCtx->fileCtx : &pGroup->pVgCtx->fileCtx;
@ -449,6 +461,10 @@ void blockDataDeepClear(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (NULL == p) {
qError("fail to get %d col from pDataBlock, numOfCols:%d", i, (int32_t)numOfCols);
continue;
}
p->pData = NULL; p->pData = NULL;
if (IS_VAR_DATA_TYPE(p->info.type)) { if (IS_VAR_DATA_TYPE(p->info.type)) {
p->varmeta.offset = NULL; p->varmeta.offset = NULL;
@ -473,8 +489,9 @@ static int32_t buildGroupCacheBaseBlock(SSDataBlock** ppDst, SSDataBlock* pSrc)
taosMemoryFree(*ppDst); taosMemoryFree(*ppDst);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info)); TAOS_MEMCPY(&(*ppDst)->info, &pSrc->info, sizeof(pSrc->info));
blockDataDeepClear(*ppDst); blockDataDeepClear(*ppDst);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -490,11 +507,17 @@ static int32_t acquireBaseBlockFromList(SGcDownstreamCtx* pCtx, SSDataBlock** pp
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static void releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) { static int32_t releaseBaseBlockToList(SGcDownstreamCtx* pCtx, SSDataBlock* pBlock) {
int32_t code = TSDB_CODE_SUCCESS;
blockDataDeepCleanup(pBlock); blockDataDeepCleanup(pBlock);
taosWLockLatch(&pCtx->blkLock); taosWLockLatch(&pCtx->blkLock);
taosArrayPush(pCtx->pFreeBlock, &pBlock); if (NULL == taosArrayPush(pCtx->pFreeBlock, &pBlock)) {
code = terrno;
}
taosWUnLockLatch(&pCtx->blkLock); taosWUnLockLatch(&pCtx->blkLock);
return code;
} }
@ -573,14 +596,15 @@ static int32_t retrieveBlkFromBufCache(SGroupCacheOperatorInfo* pGCache, SGroupC
return code; return code;
} }
taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES); QRY_ERR_RET(taosHashPut(pCache->pReadBlk, &sessionId, sizeof(sessionId), ppRes, POINTER_BYTES));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) { static FORCE_INLINE void initGcVgroupCtx(SOperatorInfo* pOperator, SGcVgroupCtx* pVgCtx, int32_t downstreamId, int32_t vgId, SArray* pTbList) {
pVgCtx->pTbList = pTbList; pVgCtx->pTbList = pTbList;
pVgCtx->id = vgId; pVgCtx->id = vgId;
snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d", (void)snprintf(pVgCtx->fileCtx.baseFilename, sizeof(pVgCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d_%d",
tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId); tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, downstreamId, vgId);
pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0; pVgCtx->fileCtx.baseFilename[sizeof(pVgCtx->fileCtx.baseFilename) - 1] = 0;
@ -594,15 +618,27 @@ static int32_t addNewGroupToVgHash(SOperatorInfo* pOperator, SSHashObj* pHash, S
if (NULL == pList) { if (NULL == pList) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pList, pNew); if (NULL == taosArrayPush(pList, pNew)) {
QRY_ERR_RET(terrno);
}
SGcVgroupCtx vgCtx = {0}; SGcVgroupCtx vgCtx = {0};
initGcVgroupCtx(pOperator, &vgCtx, pNew->pGroup->downstreamIdx, pNew->vgId, pList); initGcVgroupCtx(pOperator, &vgCtx, pNew->pGroup->downstreamIdx, pNew->vgId, pList);
tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx)); QRY_ERR_RET(tSimpleHashPut(pHash, &pNew->vgId, sizeof(pNew->vgId), &vgCtx, sizeof(vgCtx)));
pNew->pGroup->pVgCtx = tSimpleHashGet(pHash, &pNew->vgId, sizeof(pNew->vgId)); pNew->pGroup->pVgCtx = tSimpleHashGet(pHash, &pNew->vgId, sizeof(pNew->vgId));
if (NULL == pNew->pGroup->pVgCtx) {
qError("fail to get vg %d ctx from vgHash", pNew->vgId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosArrayPush(pVgCtx->pTbList, pNew); if (NULL == taosArrayPush(pVgCtx->pTbList, pNew)) {
QRY_ERR_RET(terrno);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -620,6 +656,11 @@ static FORCE_INLINE int32_t appendNewGroupToDownstream(struct SOperatorInfo* pOp
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SGcNewGroupInfo* pNew = taosArrayGet(pCtx->pNewGrpList, i); SGcNewGroupInfo* pNew = taosArrayGet(pCtx->pNewGrpList, i);
if (NULL == pNew) {
qError("fail to get vg %d SGcNewGroupInfo from pNewGrpList", i);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (!pGCache->batchFetch) { if (!pGCache->batchFetch) {
code = addNewGroupToVgHash(pOperator, pCtx->pVgTbHash, pNew); code = addNewGroupToVgHash(pOperator, pCtx->pVgTbHash, pNew);
if (code) { if (code) {
@ -672,7 +713,9 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
if (code) { if (code) {
return code; return code;
} }
taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock); if (NULL == taosArrayPush(pGCache->pDownstreams[downstreamIdx].pFreeBlock, &pGCache->pDownstreams[downstreamIdx].pBaseBlock)) {
QRY_ERR_RET(terrno);
}
} }
} }
@ -681,26 +724,36 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
return code; return code;
} }
static void notifyWaitingSessions(SArray* pWaitQueue) { static int32_t notifyWaitingSessions(SArray* pWaitQueue) {
if (NULL == pWaitQueue || taosArrayGetSize(pWaitQueue) <= 0) { if (NULL == pWaitQueue || taosArrayGetSize(pWaitQueue) <= 0) {
return; return TSDB_CODE_SUCCESS;
} }
int32_t n = taosArrayGetSize(pWaitQueue); int32_t n = taosArrayGetSize(pWaitQueue);
for (int32_t i = 0; i < n; ++i) { for (int32_t i = 0; i < n; ++i) {
SGcSessionCtx* pSession = taosArrayGetP(pWaitQueue, i); SGcSessionCtx* pSession = taosArrayGetP(pWaitQueue, i);
tsem_post(&pSession->waitSem); if (NULL == pSession) {
qError("fail to get %d SGcSessionCtx in pWaitQueue, total:%d", i, n);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
QRY_ERR_RET(tsem_post(&pSession->waitSem));
} }
return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE void handleGroupFetchDone(SGroupCacheData* pGroup) { static FORCE_INLINE int32_t handleGroupFetchDone(SGroupCacheData* pGroup) {
int32_t code = TSDB_CODE_SUCCESS;
pGroup->pBlock = NULL; pGroup->pBlock = NULL;
atomic_store_8((int8_t*)&pGroup->fetchDone, true); atomic_store_8((int8_t*)&pGroup->fetchDone, true);
taosThreadMutexLock(&pGroup->mutex); (void)taosThreadMutexLock(&pGroup->mutex);
notifyWaitingSessions(pGroup->waitQueue); code = notifyWaitingSessions(pGroup->waitQueue);
taosArrayClear(pGroup->waitQueue); taosArrayClear(pGroup->waitQueue);
taosThreadMutexUnlock(&pGroup->mutex); (void)taosThreadMutexUnlock(&pGroup->mutex);
return code;
} }
static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int32_t downstreamId, int32_t vgId) { static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int32_t downstreamId, int32_t vgId) {
@ -714,14 +767,15 @@ static int32_t addFileRefTableNum(SGcFileCacheCtx* pFileCtx, int32_t fileId, int
SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); SGroupCacheFileInfo* pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) { if (NULL == pTmp) {
sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId); (void)sprintf(&pFileCtx->baseFilename[pFileCtx->baseNameLen], "_%u", fileId);
SGroupCacheFileInfo newFile = {0}; SGroupCacheFileInfo newFile = {0};
newFile.groupNum = 1; newFile.groupNum = 1;
taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)); QRY_ERR_RET(taosHashPut(pFileCtx->pCacheFile, &fileId, sizeof(fileId), &newFile, sizeof(newFile)));
pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId)); pTmp = taosHashGet(pFileCtx->pCacheFile, &fileId, sizeof(fileId));
if (NULL == pTmp) { if (NULL == pTmp) {
return TSDB_CODE_OUT_OF_MEMORY; qError("fail to get file %d in pCacheFile", fileId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
} }
} else { } else {
pTmp->groupNum++; pTmp->groupNum++;
@ -746,7 +800,7 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat
if (NULL == pNew || pNew->uid == uid) { if (NULL == pNew || pNew->uid == uid) {
break; break;
} }
handleGroupFetchDone(pNew->pGroup); QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup));
} }
groupCacheSwitchNewFile(&pGroup->pVgCtx->fileCtx, pGroup->downstreamIdx, pGroup->vgId, true); groupCacheSwitchNewFile(&pGroup->pVgCtx->fileCtx, pGroup->downstreamIdx, pGroup->vgId, true);
@ -764,15 +818,21 @@ static int32_t handleVgroupTableFetchDone(SGcDownstreamCtx* pCtx, SGroupCacheDat
} }
static FORCE_INLINE void initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) { static FORCE_INLINE int32_t initNewGroupData(SGcDownstreamCtx* pCtx, SGroupCacheData* pGroup, int32_t downstreamIdx, int32_t vgId, bool batchFetch, bool needCache) {
taosThreadMutexInit(&pGroup->mutex, NULL); QRY_ERR_RET(taosThreadMutexInit(&pGroup->mutex, NULL));
pGroup->downstreamIdx = downstreamIdx; pGroup->downstreamIdx = downstreamIdx;
pGroup->vgId = vgId; pGroup->vgId = vgId;
pGroup->fileId = -1; pGroup->fileId = -1;
pGroup->blkList.pList = taosArrayInit(10, sizeof(SGcBlkBufBasic)); pGroup->blkList.pList = taosArrayInit(10, sizeof(SGcBlkBufBasic));
if (NULL == pGroup->blkList.pList) {
QRY_ERR_RET(terrno);
}
pGroup->startOffset = -1; pGroup->startOffset = -1;
pGroup->needCache = needCache; pGroup->needCache = needCache;
pGroup->pVgCtx = tSimpleHashGet(pCtx->pVgTbHash, &pGroup->vgId, sizeof(pGroup->vgId)); pGroup->pVgCtx = tSimpleHashGet(pCtx->pVgTbHash, &pGroup->vgId, sizeof(pGroup->vgId));
return TSDB_CODE_SUCCESS;
} }
static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* pParam, SGroupCacheData** ppGrp, int32_t vgId, int64_t uid) { static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam* pParam, SGroupCacheData** ppGrp, int32_t vgId, int64_t uid) {
@ -801,7 +861,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
if (NULL == *ppGrp) { if (NULL == *ppGrp) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache); QRY_ERR_RET(initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache));
qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache); qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
@ -829,7 +889,9 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcBlkBufInfo* pNewBlk, int64_t* pIdx) { static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcBlkBufInfo* pNewBlk, int64_t* pIdx) {
taosWLockLatch(&pGroup->blkList.lock); taosWLockLatch(&pGroup->blkList.lock);
taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic); if (NULL == taosArrayPush(pGroup->blkList.pList, &pNewBlk->basic)) {
QRY_ERR_RET(terrno);
}
*pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1; *pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1;
taosWUnLockLatch(&pGroup->blkList.lock); taosWUnLockLatch(&pGroup->blkList.lock);
@ -889,7 +951,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
pGroup->pBlock = pBlock; pGroup->pBlock = pBlock;
} }
notifyWaitingSessions(pGroup->waitQueue); QRY_ERR_RET(notifyWaitingSessions(pGroup->waitQueue));
if (pGroup == pSession->pGroupData) { if (pGroup == pSession->pGroupData) {
if (pGroup->needCache) { if (pGroup->needCache) {
pSession->lastBlkId = newBlkIdx; pSession->lastBlkId = newBlkIdx;
@ -909,7 +971,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash; SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash;
SGroupCacheData* pGroup = NULL; SGroupCacheData* pGroup = NULL;
while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) { while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) {
handleGroupFetchDone(pGroup); QRY_ERR_RET(handleGroupFetchDone(pGroup));
} }
pCtx->fetchDone = true; pCtx->fetchDone = true;
} else { } else {
@ -920,7 +982,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
uidNum = taosArrayGetSize(pVgCtx->pTbList); uidNum = taosArrayGetSize(pVgCtx->pTbList);
for (int32_t i = 0; i < uidNum; ++i) { for (int32_t i = 0; i < uidNum; ++i) {
SGcNewGroupInfo* pNew = taosArrayGet(pVgCtx->pTbList, i); SGcNewGroupInfo* pNew = taosArrayGet(pVgCtx->pTbList, i);
handleGroupFetchDone(pNew->pGroup); QRY_ERR_RET(handleGroupFetchDone(pNew->pGroup));
} }
taosArrayClear(pVgCtx->pTbList); taosArrayClear(pVgCtx->pTbList);
} }
@ -961,8 +1023,8 @@ static int32_t getCacheBlkFromDownstreamOperator(struct SOperatorInfo* pOperator
} }
SGcSessionCtx* pWaitCtx = *ppWaitCtx; SGcSessionCtx* pWaitCtx = *ppWaitCtx;
pWaitCtx->newFetch = true; pWaitCtx->newFetch = true;
taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId)); (void)taosHashRemove(pCtx->pWaitSessions, pSessionId, sizeof(*pSessionId));
tsem_post(&pWaitCtx->waitSem); QRY_ERR_RET(tsem_post(&pWaitCtx->waitSem));
return code; return code;
} }
@ -1028,36 +1090,47 @@ static int32_t groupCacheSessionWait(struct SOperatorInfo* pOperator, SGcDownstr
SGroupCacheOperatorInfo* pGCache = pOperator->info; SGroupCacheOperatorInfo* pGCache = pOperator->info;
SGroupCacheData* pGroup = pSession->pGroupData; SGroupCacheData* pGroup = pSession->pGroupData;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
bool inLock = true;
if (NULL == pGroup->waitQueue) { if (NULL == pGroup->waitQueue) {
pGroup->waitQueue = taosArrayInit(1, POINTER_BYTES); pGroup->waitQueue = taosArrayInit(1, POINTER_BYTES);
if (NULL == pGroup->waitQueue) { if (NULL == pGroup->waitQueue) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex); QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
return TSDB_CODE_OUT_OF_MEMORY;
} }
} }
taosArrayPush(pGroup->waitQueue, &pSession); if (NULL == taosArrayPush(pGroup->waitQueue, &pSession)) {
QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (!pSession->semInit) { if (!pSession->semInit) {
tsem_init(&pSession->waitSem, 0, 0); QRY_ERR_JRET(tsem_init(&pSession->waitSem, 0, 0));
pSession->semInit = true; pSession->semInit = true;
} }
taosThreadMutexUnlock(&pSession->pGroupData->mutex); (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
inLock = false;
taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES); QRY_ERR_JRET(taosHashPut(pCtx->pWaitSessions, &sessionId, sizeof(sessionId), &pSession, POINTER_BYTES));
tsem_wait(&pSession->waitSem); (void)tsem_wait(&pSession->waitSem);
if (pSession->newFetch) { if (pSession->newFetch) {
pSession->newFetch = false; pSession->newFetch = false;
return getCacheBlkFromDownstreamOperator(pOperator, pCtx, sessionId, pSession, ppRes); return getCacheBlkFromDownstreamOperator(pOperator, pCtx, sessionId, pSession, ppRes);
} }
taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId)); (void)taosHashRemove(pCtx->pWaitSessions, &sessionId, sizeof(sessionId));
bool got = false; bool got = false;
return getBlkFromSessionCacheImpl(pOperator, sessionId, pSession, ppRes, &got); return getBlkFromSessionCacheImpl(pOperator, sessionId, pSession, ppRes, &got);
_return:
if (inLock) {
(void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
}
return code;
} }
@ -1077,7 +1150,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
if ((atomic_load_64(&pCtx->fetchSessionId) == sessionId) if ((atomic_load_64(&pCtx->fetchSessionId) == sessionId)
|| (-1 == atomic_val_compare_exchange_64(&pCtx->fetchSessionId, -1, sessionId))) { || (-1 == atomic_val_compare_exchange_64(&pCtx->fetchSessionId, -1, sessionId))) {
if (locked) { if (locked) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex); (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
locked = false; locked = false;
} }
@ -1095,7 +1168,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
break; break;
} }
taosThreadMutexLock(&pSession->pGroupData->mutex); (void)taosThreadMutexLock(&pSession->pGroupData->mutex);
locked = true; locked = true;
}; };
@ -1103,7 +1176,7 @@ static int32_t getBlkFromSessionCache(struct SOperatorInfo* pOperator, int64_t s
_return: _return:
if (locked) { if (locked) {
taosThreadMutexUnlock(&pSession->pGroupData->mutex); (void)taosThreadMutexUnlock(&pSession->pGroupData->mutex);
} }
return code; return code;
@ -1165,6 +1238,10 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP
} }
*ppSession = taosHashGet(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); *ppSession = taosHashGet(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (NULL == *ppSession) {
qError("fail to get session %" PRId64 " from pSessions", pGcParam->sessionId);
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1187,15 +1264,15 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
} else if (pSession->pGroupData->needCache) { } else if (pSession->pGroupData->needCache) {
SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (ppBlock) { if (ppBlock) {
releaseBaseBlockToList(pCtx, *ppBlock); QRY_ERR_RET(releaseBaseBlockToList(pCtx, *ppBlock));
taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); (void)taosHashRemove(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
} }
} }
code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes); code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes);
if (NULL == *ppRes) { if (NULL == *ppRes) {
qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows); qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId)); (void)taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
} else { } else {
pSession->resRows += (*ppRes)->info.rows; pSession->resRows += (*ppRes)->info.rows;
qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows); qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
@ -1241,7 +1318,7 @@ static void freeRemoveGroupCacheData(void* p) {
taosArrayDestroy(pGroup->waitQueue); taosArrayDestroy(pGroup->waitQueue);
taosArrayDestroy(pGroup->blkList.pList); taosArrayDestroy(pGroup->blkList.pList);
taosThreadMutexDestroy(&pGroup->mutex); (void)taosThreadMutexDestroy(&pGroup->mutex);
qTrace("group removed"); qTrace("group removed");
} }
@ -1271,7 +1348,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
int32_t defaultVg = 0; int32_t defaultVg = 0;
SGcVgroupCtx vgCtx = {0}; SGcVgroupCtx vgCtx = {0};
initGcVgroupCtx(pOperator, &vgCtx, pCtx->id, defaultVg, NULL); initGcVgroupCtx(pOperator, &vgCtx, pCtx->id, defaultVg, NULL);
tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)); QRY_ERR_RET(tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)));
} }
pCtx->pNewGrpList = taosArrayInit(10, sizeof(SGcNewGroupInfo)); pCtx->pNewGrpList = taosArrayInit(10, sizeof(SGcNewGroupInfo));
@ -1302,7 +1379,7 @@ static int32_t initGroupCacheDownstreamCtx(SOperatorInfo* pOperator) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d", (void)snprintf(pCtx->fileCtx.baseFilename, sizeof(pCtx->fileCtx.baseFilename) - 1, "%s/gc_%d_%" PRIx64 "_%" PRIu64 "_%d",
tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id); tsTempDir, getpid(), pOperator->pTaskInfo->id.queryId, pOperator->pTaskInfo->id.taskId, pCtx->id);
pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0; pCtx->fileCtx.baseFilename[sizeof(pCtx->fileCtx.baseFilename) - 1] = 0;
pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename); pCtx->fileCtx.baseNameLen = strlen(pCtx->fileCtx.baseFilename);

View File

@ -90,7 +90,7 @@ int32_t hLeftJoinHandleSeqRowRemains(struct SOperatorInfo* pOperator, SHJoinOper
while (!allFetched) { while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched); hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) { if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL); HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL));
if (pJoin->midBlk->info.rows > 0) { if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true; pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk)); HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
@ -170,7 +170,7 @@ int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOpera
while (!allFetched) { while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched); hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) { if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL); HJ_ERR_RET(doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL));
if (pJoin->midBlk->info.rows > 0) { if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true; pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk)); HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));

View File

@ -238,10 +238,12 @@ static int32_t hJoinInitValColsInfo(SHJoinTableCtx* pTable, SNodeList* pList) {
if (NULL == pTable->valVarCols) { if (NULL == pTable->valVarCols) {
pTable->valVarCols = taosArrayInit(pTable->valNum, sizeof(int32_t)); pTable->valVarCols = taosArrayInit(pTable->valNum, sizeof(int32_t));
if (NULL == pTable->valVarCols) { if (NULL == pTable->valVarCols) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
} }
taosArrayPush(pTable->valVarCols, &i); if (NULL == taosArrayPush(pTable->valVarCols, &i)) {
return terrno;
}
} }
pTable->valCols[i].bytes = pColNode->node.resType.bytes; pTable->valCols[i].bytes = pColNode->node.resType.bytes;
if (!pTable->valCols[i].keyCol && !pTable->valCols[i].vardata) { if (!pTable->valCols[i].keyCol && !pTable->valCols[i].vardata) {
@ -332,7 +334,7 @@ static int32_t hJoinInitTableInfo(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode*
return code; return code;
} }
memcpy(&pTable->inputStat, pStat, sizeof(*pStat)); TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat));
HJ_ERR_RET(hJoinInitPrimExprCtx(pTable->primExpr, &pTable->primCtx, pTable)); HJ_ERR_RET(hJoinInitPrimExprCtx(pTable->primExpr, &pTable->primCtx, pTable));
@ -416,7 +418,9 @@ static FORCE_INLINE int32_t hJoinAddPageToBufs(SArray* pRowBufs) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(pRowBufs, &page); if (NULL == taosArrayPush(pRowBufs, &page)) {
return terrno;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -464,12 +468,21 @@ static void hJoinDestroyKeyHash(SSHashObj** ppHash) {
*ppHash = NULL; *ppHash = NULL;
} }
static FORCE_INLINE char* hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow) { static FORCE_INLINE int32_t hJoinRetrieveColDataFromRowBufs(SArray* pRowBufs, SBufRowInfo* pRow, char** ppData) {
*ppData = NULL;
if ((uint16_t)-1 == pRow->pageId) { if ((uint16_t)-1 == pRow->pageId) {
return NULL; return TSDB_CODE_SUCCESS;
} }
SBufPageInfo *pPage = taosArrayGet(pRowBufs, pRow->pageId); SBufPageInfo *pPage = taosArrayGet(pRowBufs, pRow->pageId);
return pPage->data + pRow->offset; if (NULL == pPage) {
qError("fail to get %d page, total:%d", pRow->pageId, (int32_t)taosArrayGetSize(pRowBufs));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
*ppData = pPage->data + pRow->offset;
return TSDB_CODE_SUCCESS;
} }
static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum, SBufRowInfo* pStart, SSDataBlock* pRes) { static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum, SBufRowInfo* pStart, SSDataBlock* pRes) {
@ -479,9 +492,11 @@ static int32_t hJoinCopyResRowsToBlock(SHJoinOperatorInfo* pJoin, int32_t rowNum
int32_t probeIdx = 0; int32_t probeIdx = 0;
SBufRowInfo* pRow = pStart; SBufRowInfo* pRow = pStart;
int32_t code = 0; int32_t code = 0;
char* pData = NULL;
for (int32_t r = 0; r < rowNum; ++r) { for (int32_t r = 0; r < rowNum; ++r) {
char* pData = hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow); HJ_ERR_RET(hJoinRetrieveColDataFromRowBufs(pJoin->pRowBufs, pRow, &pData));
char* pValData = pData + pBuild->valBitMapSize; char* pValData = pData + pBuild->valBitMapSize;
char* pKeyData = pProbe->keyData; char* pKeyData = pProbe->keyData;
buildIdx = buildValIdx = probeIdx = 0; buildIdx = buildValIdx = probeIdx = 0;
@ -544,7 +559,7 @@ int32_t hJoinCopyNMatchRowsToBlock(SHJoinOperatorInfo* pJoin, SSDataBlock* pRes,
SColumnInfoData* pSrc = taosArrayGet(pJoin->ctx.pProbeData->pDataBlock, pProbe->valCols[probeIdx].srcSlot); SColumnInfoData* pSrc = taosArrayGet(pJoin->ctx.pProbeData->pDataBlock, pProbe->valCols[probeIdx].srcSlot);
SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pProbe->valCols[probeIdx].dstSlot); SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pProbe->valCols[probeIdx].dstSlot);
colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows); QRY_ERR_RET(colDataAssignNRows(pDst, pRes->info.rows, pSrc, startIdx, rows));
probeIdx++; probeIdx++;
} }
@ -606,11 +621,11 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p
} }
if (pTable->keyCols[i].vardata) { if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx]; pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData)); TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData); bufLen += varDataTLen(pData);
} else { } else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx; pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes); TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes; bufLen += pTable->keyCols[i].bytes;
} }
} }
@ -627,6 +642,10 @@ bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *p
static int32_t hJoinSetKeyColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable) { static int32_t hJoinSetKeyColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) { for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (NULL == pCol) {
qError("fail to get %d col, total:%d", pTable->keyCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) { if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata); qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -654,6 +673,10 @@ static int32_t hJoinSetValColsData(SSDataBlock* pBlock, SHJoinTableCtx* pTable)
continue; continue;
} }
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->valCols[i].srcSlot); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->valCols[i].srcSlot);
if (NULL == pCol) {
qError("fail to get %d col, total:%d", pTable->valCols[i].srcSlot, (int32_t)taosArrayGetSize(pBlock->pDataBlock));
QRY_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->valCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) { if (pTable->valCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->valCols[i].srcSlot, pCol->info.type, pTable->valCols[i].vardata); qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->valCols[i].srcSlot, pCol->info.type, pTable->valCols[i].vardata);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -683,7 +706,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
char *pData = NULL; char *pData = NULL;
size_t bufLen = pTable->valBitMapSize; size_t bufLen = pTable->valBitMapSize;
memset(pTable->valData, 0, pTable->valBitMapSize); TAOS_MEMSET(pTable->valData, 0, pTable->valBitMapSize);
for (int32_t i = 0, m = 0; i < pTable->valNum; ++i) { for (int32_t i = 0, m = 0; i < pTable->valNum; ++i) {
if (pTable->valCols[i].keyCol) { if (pTable->valCols[i].keyCol) {
continue; continue;
@ -693,7 +716,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
colDataSetNull_f(pTable->valData, m); colDataSetNull_f(pTable->valData, m);
} else { } else {
pData = pTable->valCols[i].data + pTable->valCols[i].offset[rowIdx]; pData = pTable->valCols[i].data + pTable->valCols[i].offset[rowIdx];
memcpy(pTable->valData + bufLen, pData, varDataTLen(pData)); TAOS_MEMCPY(pTable->valData + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData); bufLen += varDataTLen(pData);
} }
} else { } else {
@ -701,7 +724,7 @@ static FORCE_INLINE void hJoinCopyValColsDataToBuf(SHJoinTableCtx* pTable, int32
colDataSetNull_f(pTable->valData, m); colDataSetNull_f(pTable->valData, m);
} else { } else {
pData = pTable->valCols[i].data + pTable->valCols[i].bytes * rowIdx; pData = pTable->valCols[i].data + pTable->valCols[i].bytes * rowIdx;
memcpy(pTable->valData + bufLen, pData, pTable->valCols[i].bytes); TAOS_MEMCPY(pTable->valData + bufLen, pData, pTable->valCols[i].bytes);
bufLen += pTable->valCols[i].bytes; bufLen += pTable->valCols[i].bytes;
} }
} }
@ -1009,7 +1032,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) {
} }
if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) { if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) {
doFilter(pRes, pJoin->pFinFilter, NULL); code = doFilter(pRes, pJoin->pFinFilter, NULL);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
} }
if (pRes->info.rows > 0) { if (pRes->info.rows > 0) {
return pRes; return pRes;
@ -1037,7 +1064,11 @@ static SSDataBlock* hJoinMainProcess(struct SOperatorInfo* pOperator) {
} }
if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) { if (pRes->info.rows > 0 && pJoin->pFinFilter != NULL) {
doFilter(pRes, pJoin->pFinFilter, NULL); code = doFilter(pRes, pJoin->pFinFilter, NULL);
if (code) {
pTaskInfo->code = code;
T_LONG_JMP(pTaskInfo->env, code);
}
} }
if (pRes->info.rows > 0) { if (pRes->info.rows > 0) {
@ -1119,13 +1150,25 @@ static uint32_t hJoinGetFinBlkCapacity(SHJoinOperatorInfo* pJoin, SHashJoinPhysi
int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinNode) { int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinNode) {
pJoin->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); pJoin->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode)); int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
}
if (NULL != pJoin->pPreFilter) { if (NULL != pJoin->pPreFilter) {
pJoin->midBlk = createOneDataBlock(pJoin->finBlk, false); pJoin->midBlk = createOneDataBlock(pJoin->finBlk, false);
blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity); if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
}
code = blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity);
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
}
} }
pJoin->blkThreshold = pJoin->finBlk->info.capacity * HJOIN_BLK_THRESHOLD_RATIO; pJoin->blkThreshold = pJoin->finBlk->info.capacity * HJOIN_BLK_THRESHOLD_RATIO;
@ -1152,8 +1195,8 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n
setOperatorInfo(pOperator, "HashJoinOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); setOperatorInfo(pOperator, "HashJoinOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo);
hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]); HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0]));
hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]); HJ_ERR_JRET(hJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1]));
hJoinSetBuildAndProbeTable(pInfo, pJoinNode); hJoinSetBuildAndProbeTable(pInfo, pJoinNode);
@ -1183,6 +1226,7 @@ SOperatorInfo* createHashJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t n
return pOperator; return pOperator;
_return: _return:
if (pInfo != NULL) { if (pInfo != NULL) {
destroyHashJoinOperator(pInfo); destroyHashJoinOperator(pInfo);
} }

View File

@ -72,9 +72,15 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
if ((!pCtx->seqWinGrp) && 0 == cache->grpIdx && probeRows * buildTotalRows <= rowsLeft) { if ((!pCtx->seqWinGrp) && 0 == cache->grpIdx && probeRows * buildTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(cache->grps, 0); SMJoinGrpRows* pFirstBuild = taosArrayGet(cache->grps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) { if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; cache->grpIdx < buildGrpNum; ++cache->grpIdx) { for (; cache->grpIdx < buildGrpNum; ++cache->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx; buildGrp->readIdx = buildGrp->beginIdx;
} }
@ -89,6 +95,9 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx; probeGrp->endIdx = probeGrp->readIdx;
for (; cache->grpIdx < buildGrpNum && rowsLeft > 0; ++cache->grpIdx) { for (; cache->grpIdx < buildGrpNum && rowsLeft > 0; ++cache->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, cache->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -134,7 +143,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (build->grpRowIdx >= 0) { if (build->grpRowIdx >= 0) {
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (build->grpRowIdx < 0) { if (build->grpRowIdx < 0) {
probeGrp->readIdx++; probeGrp->readIdx++;
} }
@ -171,7 +181,8 @@ static int32_t mOuterJoinHashFullCart(SMJoinMergeCtx* pCtx) {
} }
build->grpRowIdx = 0; build->grpRowIdx = 0;
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (!contLoop) { if (!contLoop) {
if (build->grpRowIdx < 0) { if (build->grpRowIdx < 0) {
probeGrp->readIdx++; probeGrp->readIdx++;
@ -193,15 +204,27 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) { if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0); SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) { if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; build->grpIdx < buildGrpNum; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx; buildGrp->readIdx = buildGrp->beginIdx;
} }
@ -215,6 +238,9 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx; probeGrp->endIdx = probeGrp->readIdx;
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -254,6 +280,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
int32_t rowsLeft = pCtx->midBlk->info.capacity; int32_t rowsLeft = pCtx->midBlk->info.capacity;
@ -274,6 +304,10 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (startRowIdx < 0) { if (startRowIdx < 0) {
startRowIdx = buildGrp->readIdx; startRowIdx = buildGrp->readIdx;
} }
@ -351,13 +385,17 @@ static int32_t mOuterJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, bool* contLoop)
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t startRowIdx = 0; int32_t startRowIdx = 0;
//blockDataCleanup(pCtx->midBlk); //blockDataCleanup(pCtx->midBlk);
do { do {
startRowIdx = build->grpRowIdx; startRowIdx = build->grpRowIdx;
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) { if (pCtx->midBlk->info.rows > 0) {
if (build->rowBitmapSize > 0) { if (build->rowBitmapSize > 0) {
@ -406,6 +444,10 @@ static int32_t mOuterJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
bool contLoop = false; bool contLoop = false;
if (build->grpRowIdx >= 0) { if (build->grpRowIdx >= 0) {
@ -500,7 +542,15 @@ static bool mLeftJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoi
if (buildGot) { if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
if (NULL == pProbeCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false; buildGot = false;
@ -649,15 +699,27 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) { if (0 == build->grpIdx && probeRows * build->grpTotalRows <= rowsLeft) {
SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0); SMJoinGrpRows* pFirstBuild = taosArrayGet(build->eqGrps, 0);
if (NULL == pFirstBuild) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pFirstBuild->readIdx == pFirstBuild->beginIdx) { if (pFirstBuild->readIdx == pFirstBuild->beginIdx) {
for (; build->grpIdx < buildGrpNum; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
buildGrp->readIdx = buildGrp->beginIdx; buildGrp->readIdx = buildGrp->beginIdx;
} }
@ -671,6 +733,9 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) {
probeGrp->endIdx = probeGrp->readIdx; probeGrp->endIdx = probeGrp->readIdx;
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp));
@ -711,9 +776,13 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (build->grpRowIdx >= 0) { if (build->grpRowIdx >= 0) {
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (build->grpRowIdx < 0) { if (build->grpRowIdx < 0) {
probeGrp->readIdx++; probeGrp->readIdx++;
} }
@ -734,7 +803,8 @@ static int32_t mInnerJoinHashCart(SMJoinMergeCtx* pCtx) {
if (NULL != pGrp) { if (NULL != pGrp) {
build->pHashCurGrp = *pGrp; build->pHashCurGrp = *pGrp;
build->grpRowIdx = 0; build->grpRowIdx = 0;
bool contLoop = mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); bool contLoop = false;
MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, &contLoop));
if (!contLoop) { if (!contLoop) {
if (build->grpRowIdx < 0) { if (build->grpRowIdx < 0) {
probeGrp->readIdx++; probeGrp->readIdx++;
@ -772,7 +842,15 @@ static bool mInnerJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJo
if (buildGot) { if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
if (NULL == pProbeCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false; buildGot = false;
@ -903,6 +981,10 @@ static int32_t mFullJoinMergeCart(SMJoinMergeCtx* pCtx) {
static FORCE_INLINE int32_t mFullJoinOutputHashRow(SMJoinMergeCtx* pCtx, SMJoinHashGrpRows* pGrpRows, int32_t idx) { static FORCE_INLINE int32_t mFullJoinOutputHashRow(SMJoinMergeCtx* pCtx, SMJoinHashGrpRows* pGrpRows, int32_t idx) {
SMJoinGrpRows grp = {0}; SMJoinGrpRows grp = {0};
SMJoinRowPos* pPos = taosArrayGet(pGrpRows->pRows, idx); SMJoinRowPos* pPos = taosArrayGet(pGrpRows->pRows, idx);
if (NULL == pPos) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
grp.blk = pPos->pBlk; grp.blk = pPos->pBlk;
grp.readIdx = pPos->pos; grp.readIdx = pPos->pos;
grp.endIdx = pPos->pos; grp.endIdx = pPos->pos;
@ -1042,6 +1124,10 @@ static int32_t mFullJoinHandleMergeGrpRemains(SMJoinMergeCtx* pCtx) {
grpDone = false; grpDone = false;
SMJoinGrpRows* pGrpRows = taosArrayGet(build->eqGrps, pNMatch->grpIdx); SMJoinGrpRows* pGrpRows = taosArrayGet(build->eqGrps, pNMatch->grpIdx);
if (NULL == pGrpRows) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrpRows->allRowsMatch) { if (pGrpRows->allRowsMatch) {
continue; continue;
} }
@ -1283,7 +1369,7 @@ static int32_t mSemiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p
do { do {
blockDataCleanup(pCtx->midBlk); blockDataCleanup(pCtx->midBlk);
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) { if (pCtx->midBlk->info.rows > 0) {
MJ_ERR_RET(mJoinFilterAndKeepSingleRow(pCtx->midBlk, pCtx->pJoin->pPreFilter)); MJ_ERR_RET(mJoinFilterAndKeepSingleRow(pCtx->midBlk, pCtx->pJoin->pPreFilter));
@ -1312,6 +1398,9 @@ static int32_t mSemiJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0; size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
@ -1343,6 +1432,10 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0; size_t bufLen = 0;
for (; !GRP_DONE(probeGrp) && !BLK_IS_FULL(pCtx->finBlk); ++probeGrp->readIdx) { for (; !GRP_DONE(probeGrp) && !BLK_IS_FULL(pCtx->finBlk); ++probeGrp->readIdx) {
@ -1358,7 +1451,7 @@ static int32_t mSemiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
build->pHashCurGrp = *(SArray**)pGrp; build->pHashCurGrp = *(SArray**)pGrp;
ASSERT(1 == taosArrayGetSize(build->pHashCurGrp)); ASSERT(1 == taosArrayGetSize(build->pHashCurGrp));
build->grpRowIdx = 0; build->grpRowIdx = 0;
mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build); MJ_ERR_RET(mJoinHashGrpCart(pCtx->finBlk, probeGrp, true, probe, build, NULL));
ASSERT(build->grpRowIdx < 0); ASSERT(build->grpRowIdx < 0);
} }
@ -1372,6 +1465,10 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* buildGrp = NULL; SMJoinGrpRows* buildGrp = NULL;
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
@ -1387,6 +1484,9 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
blockDataCleanup(pCtx->midBlk); blockDataCleanup(pCtx->midBlk);
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
@ -1448,6 +1548,10 @@ static int32_t mSemiJoinMergeFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, 0); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, 0);
if (NULL == buildGrp || NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
int32_t probeRows = GRP_REMAIN_ROWS(probeGrp); int32_t probeRows = GRP_REMAIN_ROWS(probeGrp);
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
@ -1584,6 +1688,10 @@ static int32_t mAntiJoinHashFullCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0; size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
@ -1616,7 +1724,7 @@ static int32_t mAntiJoinHashGrpCartFilter(SMJoinMergeCtx* pCtx, SMJoinGrpRows* p
do { do {
blockDataCleanup(pCtx->midBlk); blockDataCleanup(pCtx->midBlk);
mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build); MJ_ERR_RET(mJoinHashGrpCart(pCtx->midBlk, probeGrp, true, probe, build, NULL));
if (pCtx->midBlk->info.rows > 0) { if (pCtx->midBlk->info.rows > 0) {
MJ_ERR_RET(mJoinFilterAndNoKeepRows(pCtx->midBlk, pCtx->pJoin->pPreFilter)); MJ_ERR_RET(mJoinFilterAndNoKeepRows(pCtx->midBlk, pCtx->pJoin->pPreFilter));
@ -1642,6 +1750,10 @@ static int32_t mAntiJoinHashSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, 0);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
size_t bufLen = 0; size_t bufLen = 0;
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
@ -1682,6 +1794,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
SMJoinTableCtx* probe = pCtx->pJoin->probe; SMJoinTableCtx* probe = pCtx->pJoin->probe;
SMJoinTableCtx* build = pCtx->pJoin->build; SMJoinTableCtx* build = pCtx->pJoin->build;
SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx); SMJoinGrpRows* probeGrp = taosArrayGet(probe->eqGrps, probe->grpIdx);
if (NULL == probeGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* buildGrp = NULL; SMJoinGrpRows* buildGrp = NULL;
int32_t buildGrpNum = taosArrayGetSize(build->eqGrps); int32_t buildGrpNum = taosArrayGetSize(build->eqGrps);
int32_t probeEndIdx = probeGrp->endIdx; int32_t probeEndIdx = probeGrp->endIdx;
@ -1697,6 +1813,10 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) {
blockDataCleanup(pCtx->midBlk); blockDataCleanup(pCtx->midBlk);
for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) { for (; build->grpIdx < buildGrpNum && rowsLeft > 0; ++build->grpIdx) {
buildGrp = taosArrayGet(build->eqGrps, build->grpIdx); buildGrp = taosArrayGet(build->eqGrps, build->grpIdx);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) { if (rowsLeft >= GRP_REMAIN_ROWS(buildGrp)) {
MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp));
rowsLeft -= GRP_REMAIN_ROWS(buildGrp); rowsLeft -= GRP_REMAIN_ROWS(buildGrp);
@ -1889,6 +2009,9 @@ int32_t mAsofBackwardAddEqRowsToCache(struct SOperatorInfo* pOperator, SMJoinWin
grp.blk = pTable->blk; grp.blk = pTable->blk;
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) { if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2102,7 +2225,9 @@ int32_t mAsofBackwardHandleGrpRemains(SMJoinWindowCtx* pCtx) {
return (pCtx->lastEqGrp) ? mAsofBackwardDumpUpdateEqRows(pCtx, pCtx->pJoin, false, true) : mAsofBackwardDumpGrpCache(pCtx); return (pCtx->lastEqGrp) ? mAsofBackwardDumpUpdateEqRows(pCtx, pCtx->pJoin, false, true) : mAsofBackwardDumpGrpCache(pCtx);
} }
static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) { static int32_t mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe); bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false; bool buildGot = false;
@ -2116,7 +2241,7 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo*
mJoinSetDone(pOperator); mJoinSetDone(pOperator);
} }
return false; return TSDB_CODE_SUCCESS;
} }
break; break;
@ -2124,13 +2249,18 @@ static bool mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo*
if (buildGot && NULL == pCtx->cache.outBlk) { if (buildGot && NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false); pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit); if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
} }
pCtx->probeGrp.blk = pJoin->probe->blk; pCtx->probeGrp.blk = pJoin->probe->blk;
pCtx->buildGrp.blk = pJoin->build->blk; pCtx->buildGrp.blk = pJoin->build->blk;
return true; *newBlock = true;
return TSDB_CODE_SUCCESS;
} }
@ -2142,6 +2272,7 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) {
int64_t buildTs = 0; int64_t buildTs = 0;
SColumnInfoData* pBuildCol = NULL; SColumnInfoData* pBuildCol = NULL;
SColumnInfoData* pProbeCol = NULL; SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk); blockDataCleanup(pCtx->finBlk);
@ -2154,7 +2285,8 @@ SSDataBlock* mAsofBackwardJoinDo(struct SOperatorInfo* pOperator) {
} }
do { do {
if (!mAsofBackwardRetrieve(pOperator, pJoin, pCtx)) { MJ_ERR_JRET(mAsofBackwardRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) { if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue; continue;
} }
@ -2240,6 +2372,10 @@ int32_t mAsofForwardTrimCacheBlk(SMJoinWindowCtx* pCtx) {
} }
SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0); SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrp->blk == pCtx->cache.outBlk && pCtx->pJoin->build->blkRowIdx > 0) { if (pGrp->blk == pCtx->cache.outBlk && pCtx->pJoin->build->blkRowIdx > 0) {
MJ_ERR_RET(blockDataTrimFirstRows(pGrp->blk, pCtx->pJoin->build->blkRowIdx)); MJ_ERR_RET(blockDataTrimFirstRows(pGrp->blk, pCtx->pJoin->build->blkRowIdx));
pCtx->pJoin->build->blkRowIdx = 0; pCtx->pJoin->build->blkRowIdx = 0;
@ -2262,6 +2398,10 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps); int32_t grpNum = taosArrayGetSize(pCache->grps);
if (grpNum >= 1) { if (grpNum >= 1) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, grpNum - 1); SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, grpNum - 1);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pGrp->blk != pCache->outBlk) { if (pGrp->blk != pCache->outBlk) {
int32_t beginIdx = (1 == grpNum) ? build->blkRowIdx : 0; int32_t beginIdx = (1 == grpNum) ? build->blkRowIdx : 0;
MJ_ERR_RET(blockDataMergeNRows(pCache->outBlk, pGrp->blk, beginIdx, pGrp->blk->info.rows - beginIdx)); MJ_ERR_RET(blockDataMergeNRows(pCache->outBlk, pGrp->blk, beginIdx, pGrp->blk->info.rows - beginIdx));
@ -2271,8 +2411,12 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
pGrp->readIdx = 0; pGrp->readIdx = 0;
//pGrp->endIdx = pGrp->blk->info.rows - 1; //pGrp->endIdx = pGrp->blk->info.rows - 1;
} else { } else {
taosArrayPop(pCache->grps); (void)taosArrayPop(pCache->grps);
pGrp = taosArrayGet(pCache->grps, 0); pGrp = taosArrayGet(pCache->grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
ASSERT(pGrp->blk == pCache->outBlk); ASSERT(pGrp->blk == pCache->outBlk);
//pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx; //pGrp->endIdx = pGrp->blk->info.rows - pGrp->beginIdx;
} }
@ -2311,16 +2455,20 @@ int32_t mAsofForwardChkFillGrpCache(SMJoinWindowCtx* pCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) { int32_t mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCtx->cache.grps); int32_t grpNum = taosArrayGetSize(pCtx->cache.grps);
if (grpNum <= 0) { if (grpNum <= 0) {
return; return TSDB_CODE_SUCCESS;
} }
SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0); SMJoinGrpRows* pGrp = taosArrayGet(pCtx->cache.grps, 0);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (1 == grpNum) { if (1 == grpNum) {
pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows - pGrp->beginIdx, pCtx->jLimit) - 1; pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows - pGrp->beginIdx, pCtx->jLimit) - 1;
return; return TSDB_CODE_SUCCESS;
} }
ASSERT(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx)); ASSERT(pCtx->jLimit > (pGrp->blk->info.rows - pGrp->beginIdx));
@ -2329,7 +2477,13 @@ void mAsofForwardUpdateBuildGrpEndIdx(SMJoinWindowCtx* pCtx) {
int64_t remainRows = pCtx->jLimit - (pGrp->endIdx - pGrp->beginIdx + 1); int64_t remainRows = pCtx->jLimit - (pGrp->endIdx - pGrp->beginIdx + 1);
pGrp = taosArrayGet(pCtx->cache.grps, 1); pGrp = taosArrayGet(pCtx->cache.grps, 1);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows, remainRows) - 1; pGrp->endIdx = pGrp->beginIdx + TMIN(pGrp->blk->info.rows, remainRows) - 1;
return TSDB_CODE_SUCCESS;
} }
int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) { int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) {
@ -2338,13 +2492,16 @@ int32_t mAsofForwardFillDumpGrpCache(SMJoinWindowCtx* pCtx, bool lastBuildGrp) {
MJ_ERR_RET(mAsofForwardChkFillGrpCache(pCtx)); MJ_ERR_RET(mAsofForwardChkFillGrpCache(pCtx));
} }
mAsofForwardUpdateBuildGrpEndIdx(pCtx); MJ_ERR_RET(mAsofForwardUpdateBuildGrpEndIdx(pCtx));
return mWinJoinDumpGrpCache(pCtx); return mWinJoinDumpGrpCache(pCtx);
} }
int32_t mAsofForwardSkipEqRows(SMJoinWindowCtx* pCtx, SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk) { int32_t mAsofForwardSkipEqRows(SMJoinWindowCtx* pCtx, SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) { if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
*wholeBlk = false; *wholeBlk = false;
@ -2479,7 +2636,9 @@ int32_t mAsofForwardSkipBuildGrp(SMJoinWindowCtx* pCtx, SMJoinOperatorInfo* pJoi
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) { static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe); bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false; bool buildGot = false;
@ -2496,12 +2655,16 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
mJoinSetDone(pOperator); mJoinSetDone(pOperator);
} }
return false; return TSDB_CODE_SUCCESS;
} }
if (buildGot) { if (buildGot) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pProbeCol || NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
MJOIN_POP_TB_BLK(&pCtx->cache); MJOIN_POP_TB_BLK(&pCtx->cache);
@ -2516,7 +2679,10 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
if (buildGot && pJoin->build->newBlk) { if (buildGot && pJoin->build->newBlk) {
if (NULL == pCtx->cache.outBlk) { if (NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false); pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit); if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
} }
MJOIN_PUSH_BLK_TO_CACHE(&pCtx->cache, pJoin->build->blk); MJOIN_PUSH_BLK_TO_CACHE(&pCtx->cache, pJoin->build->blk);
@ -2524,8 +2690,9 @@ static bool mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* p
} }
pCtx->probeGrp.blk = pJoin->probe->blk; pCtx->probeGrp.blk = pJoin->probe->blk;
*newBlock = true;
return true; return TSDB_CODE_SUCCESS;
} }
@ -2537,6 +2704,7 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) {
int64_t buildTs = 0; int64_t buildTs = 0;
SColumnInfoData* pBuildCol = NULL; SColumnInfoData* pBuildCol = NULL;
SColumnInfoData* pProbeCol = NULL; SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk); blockDataCleanup(pCtx->finBlk);
@ -2549,7 +2717,8 @@ SSDataBlock* mAsofForwardJoinDo(struct SOperatorInfo* pOperator) {
} }
do { do {
if (!mAsofForwardRetrieve(pOperator, pJoin, pCtx)) { MJ_ERR_JRET(mAsofForwardRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) { if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue; continue;
} }
@ -2643,7 +2812,7 @@ static FORCE_INLINE void mWinJoinPopFrontGroup(SMJoinWindowCtx* pCtx, SMJoinGrpR
if (pGrp->blk == pCtx->cache.outBlk) { if (pGrp->blk == pCtx->cache.outBlk) {
blockDataCleanup(pGrp->blk); blockDataCleanup(pGrp->blk);
} else if (pGrp->clonedBlk) { } else if (pGrp->clonedBlk) {
blockDataDestroy(pGrp->blk); (void)blockDataDestroy(pGrp->blk);
} }
taosArrayPopFrontBatch(pCtx->cache.grps, 1); taosArrayPopFrontBatch(pCtx->cache.grps, 1);
@ -2658,6 +2827,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
} }
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGetLast(pGrpArray); SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGetLast(pGrpArray);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (!pGrp->clonedBlk) { if (!pGrp->clonedBlk) {
if (0 == pGrp->beginIdx) { if (0 == pGrp->beginIdx) {
pGrp->blk = createOneDataBlock(pGrp->blk, true); pGrp->blk = createOneDataBlock(pGrp->blk, true);
@ -2667,6 +2840,10 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
pGrp->beginIdx = 0; pGrp->beginIdx = 0;
pGrp->readIdx = 0; pGrp->readIdx = 0;
} }
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
pGrp->clonedBlk = true; pGrp->clonedBlk = true;
} }
@ -2674,14 +2851,16 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx) { static int32_t mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin, SMJoinWindowCtx* pCtx, bool* newBlock) {
*newBlock = false;
bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe); bool probeGot = mJoinRetrieveBlk(pJoin, &pJoin->probe->blkRowIdx, &pJoin->probe->blk, pJoin->probe);
bool buildGot = false; bool buildGot = false;
do { do {
if (probeGot || MJOIN_DS_NEED_INIT(pOperator, pJoin->build)) { if (probeGot || MJOIN_DS_NEED_INIT(pOperator, pJoin->build)) {
if (NULL == pJoin->build->blk) { if (NULL == pJoin->build->blk) {
mWinJoinCloneCacheBlk(pCtx); MJ_ERR_RET(mWinJoinCloneCacheBlk(pCtx));
} }
buildGot = mJoinRetrieveBlk(pJoin, &pJoin->build->blkRowIdx, &pJoin->build->blk, pJoin->build); buildGot = mJoinRetrieveBlk(pJoin, &pJoin->build->blkRowIdx, &pJoin->build->blk, pJoin->build);
@ -2692,12 +2871,16 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin
mJoinSetDone(pOperator); mJoinSetDone(pOperator);
} }
return false; return TSDB_CODE_SUCCESS;
} }
if (buildGot && pCtx->forwardRowsAcq) { if (buildGot && pCtx->forwardRowsAcq) {
SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId); SColumnInfoData* pProbeCol = taosArrayGet(pJoin->probe->blk->pDataBlock, pJoin->probe->primCtx.targetSlotId);
SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId); SColumnInfoData* pBuildCol = taosArrayGet(pJoin->build->blk->pDataBlock, pJoin->build->primCtx.targetSlotId);
if (NULL == pProbeCol || NULL == pBuildCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) { if (MJOIN_BUILD_BLK_OOR(pCtx->ascTs, pProbeCol->pData, pJoin->probe->blkRowIdx, pBuildCol->pData, pJoin->build->blk->info.rows)) {
pJoin->build->blkRowIdx = pJoin->build->blk->info.rows; pJoin->build->blkRowIdx = pJoin->build->blk->info.rows;
buildGot = false; buildGot = false;
@ -2709,13 +2892,18 @@ static bool mWinJoinRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo* pJoin
} while (true); } while (true);
pCtx->probeGrp.blk = pJoin->probe->blk; pCtx->probeGrp.blk = pJoin->probe->blk;
*newBlock = true;
return true;
return TSDB_CODE_SUCCESS;
} }
int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) { int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) {
SSDataBlock* pBlk = build->blk; SSDataBlock* pBlk = build->blk;
SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pCtx->ascTs) { if (pCtx->ascTs) {
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) < pCtx->winBeginTs) { if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) < pCtx->winBeginTs) {
*winEnd = false; *winEnd = false;
@ -2736,6 +2924,9 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache,
if (*((int64_t*)pCol->pData + build->blkRowIdx) <= pCtx->winEndTs) { if (*((int64_t*)pCol->pData + build->blkRowIdx) <= pCtx->winEndTs) {
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx}; SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx;
@ -2772,7 +2963,10 @@ int32_t mWinJoinTryAddWinBeginBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache,
if (*((int64_t*)pCol->pData + build->blkRowIdx) >= pCtx->winBeginTs) { if (*((int64_t*)pCol->pData + build->blkRowIdx) >= pCtx->winBeginTs) {
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx}; SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx;
@ -2841,7 +3035,15 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps); int32_t grpNum = taosArrayGetSize(pCache->grps);
for (int32_t i = 0; i < grpNum; ++i) { for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i); SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) < pCtx->winBeginTs) { if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) < pCtx->winBeginTs) {
mWinJoinPopFrontGroup(pCtx, pGrp); mWinJoinPopFrontGroup(pCtx, pGrp);
grpNum--; grpNum--;
@ -2877,6 +3079,7 @@ int32_t mWinJoinMoveAscWinBegin(SMJoinWindowCtx* pCtx) {
pCache->grps = pCache->grpsQueue; pCache->grps = pCache->grpsQueue;
pCache->rowNum = 1; pCache->rowNum = 1;
pCache->grpsQueue = NULL; pCache->grpsQueue = NULL;
continue; continue;
} }
@ -2893,7 +3096,15 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) {
int32_t grpNum = taosArrayGetSize(pCache->grps); int32_t grpNum = taosArrayGetSize(pCache->grps);
for (int32_t i = 0; i < grpNum; ++i) { for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i); SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) > pCtx->winEndTs) { if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) > pCtx->winEndTs) {
mWinJoinPopFrontGroup(pCtx, pGrp); mWinJoinPopFrontGroup(pCtx, pGrp);
@ -2930,6 +3141,7 @@ int32_t mWinJoinMoveDescWinBegin(SMJoinWindowCtx* pCtx) {
pCache->grps = pCache->grpsQueue; pCache->grps = pCache->grpsQueue;
pCache->rowNum = 1; pCache->rowNum = 1;
pCache->grpsQueue = NULL; pCache->grpsQueue = NULL;
continue; continue;
} }
@ -2963,6 +3175,10 @@ void mWinJoinRemoveOverflowGrp(SMJoinWindowCtx* pCtx) {
int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) { int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, SMJoinTableCtx* build, bool* winEnd) {
SSDataBlock* pBlk = build->blk; SSDataBlock* pBlk = build->blk;
SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pBlk->pDataBlock, build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx}; SMJoinGrpRows grp = {.blk = pBlk, .beginIdx = build->blkRowIdx};
if (pCtx->ascTs) { if (pCtx->ascTs) {
@ -2978,6 +3194,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) <= pCtx->winEndTs) { if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) <= pCtx->winEndTs) {
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pBlk->info.rows - 1; pGrp->endIdx = pBlk->info.rows - 1;
@ -3005,6 +3224,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
} }
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = build->blkRowIdx - 1; pGrp->endIdx = build->blkRowIdx - 1;
@ -3025,6 +3247,9 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) >= pCtx->winBeginTs) { if (*((int64_t*)pCol->pData + pBlk->info.rows - 1) >= pCtx->winBeginTs) {
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pBlk->info.rows - 1; pGrp->endIdx = pBlk->info.rows - 1;
@ -3047,7 +3272,10 @@ int32_t mWinJoinTryAddWinEndBlk(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache, S
} }
SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp); SMJoinGrpRows* pGrp = taosArrayPush(pCache->grps, &grp);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = build->blkRowIdx - 1; pGrp->endIdx = build->blkRowIdx - 1;
@ -3107,7 +3335,15 @@ int32_t mWinJoinMoveAscWinEnd(SMJoinWindowCtx* pCtx) {
} }
SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps); SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) <= pCtx->winEndTs) { if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) <= pCtx->winEndTs) {
pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1; pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1;
if (pCache->rowNum >= pCtx->jLimit) { if (pCache->rowNum >= pCtx->jLimit) {
@ -3150,7 +3386,15 @@ int32_t mWinJoinMoveDescWinEnd(SMJoinWindowCtx* pCtx) {
} }
SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps); SMJoinGrpRows* pGrp = taosArrayGetLast(pCache->grps);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pGrp->blk->pDataBlock, pCtx->pJoin->build->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) >= pCtx->winBeginTs) { if (*((int64_t*)pCol->pData + pGrp->blk->info.rows - 1) >= pCtx->winBeginTs) {
pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1; pCache->rowNum += pGrp->blk->info.rows - pGrp->endIdx - 1;
pGrp->endIdx = pGrp->blk->info.rows - 1; pGrp->endIdx = pGrp->blk->info.rows - 1;
@ -3194,6 +3438,10 @@ int32_t mWinJoinTrimDumpGrpCache(SMJoinWindowCtx* pCtx) {
int32_t buildGrpNum = taosArrayGetSize(cache->grps); int32_t buildGrpNum = taosArrayGetSize(cache->grps);
for (int32_t i = 0; i < buildGrpNum && skipRows > 0; ++i) { for (int32_t i = 0; i < buildGrpNum && skipRows > 0; ++i) {
SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, i); SMJoinGrpRows* buildGrp = taosArrayGet(cache->grps, i);
if (NULL == buildGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (skipRows >= GRP_REMAIN_ROWS(buildGrp)) { if (skipRows >= GRP_REMAIN_ROWS(buildGrp)) {
skipRows -= GRP_REMAIN_ROWS(buildGrp); skipRows -= GRP_REMAIN_ROWS(buildGrp);
mWinJoinPopFrontGroup(pCtx, buildGrp); mWinJoinPopFrontGroup(pCtx, buildGrp);
@ -3220,6 +3468,7 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int64_t probeTs = 0; int64_t probeTs = 0;
SColumnInfoData* pProbeCol = NULL; SColumnInfoData* pProbeCol = NULL;
bool newBlock = false;
blockDataCleanup(pCtx->finBlk); blockDataCleanup(pCtx->finBlk);
@ -3232,7 +3481,8 @@ SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator) {
} }
do { do {
if (!mWinJoinRetrieve(pOperator, pJoin, pCtx)) { MJ_ERR_JRET(mWinJoinRetrieve(pOperator, pJoin, pCtx, &newBlock));
if (!newBlock) {
if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) { if (pCtx->groupJoin && pCtx->finBlk->info.rows <= 0 && !mJoinIsDone(pOperator)) {
continue; continue;
} }
@ -3293,7 +3543,7 @@ int32_t mJoinInitWindowCache(SMJoinWinCache* pCache, SMJoinOperatorInfo* pJoin,
pCache->grps = taosArrayInit(2, sizeof(SMJoinGrpRows)); pCache->grps = taosArrayInit(2, sizeof(SMJoinGrpRows));
if (NULL == pCache->grps) { if (NULL == pCache->grps) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
//taosArrayReserve(pTable->eqGrps, 1); //taosArrayReserve(pTable->eqGrps, 1);
@ -3369,7 +3619,11 @@ int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* p
} }
pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)); if (NULL == pCtx->finBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO; pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO;
@ -3409,13 +3663,20 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ
} }
pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); pCtx->finBlk = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
if (NULL == pCtx->finBlk) {
MJ_ERR_RET(terrno);
}
ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0); ASSERT(pJoinNode->node.pOutputDataBlockDesc->totalRowSize > 0);
blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)); MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
if (pJoin->pFPreFilter) { if (pJoin->pFPreFilter) {
pCtx->midBlk = createOneDataBlock(pCtx->finBlk, false); pCtx->midBlk = createOneDataBlock(pCtx->finBlk, false);
blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity); if (NULL == pCtx->midBlk) {
MJ_ERR_RET(terrno);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity));
} }
pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO; pCtx->blkThreshold = pCtx->finBlk->info.capacity * MJOIN_BLK_THRESHOLD_RATIO;

View File

@ -30,6 +30,9 @@
int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp) { int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
pGrp->beginIdx = pTable->blkRowIdx; pGrp->beginIdx = pTable->blkRowIdx;
pGrp->readIdx = pTable->blkRowIdx; pGrp->readIdx = pTable->blkRowIdx;
@ -59,12 +62,16 @@ int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBl
} }
void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) { int32_t mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
int32_t bmLen = BitmapLen(pBlock->info.rows); int32_t bmLen = BitmapLen(pBlock->info.rows);
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
if (NULL == pDst) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
// it is a reserved column for scalar function, and no data in this column yet. // it is a reserved column for scalar function, and no data in this column yet.
if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) { if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) {
continue; continue;
@ -86,7 +93,7 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
} else { } else {
bool isNull = colDataIsNull_f(pDst->nullbitmap, 0); bool isNull = colDataIsNull_f(pDst->nullbitmap, 0);
memset(pDst->nullbitmap, 0, bmLen); TAOS_MEMSET(pDst->nullbitmap, 0, bmLen);
if (isNull) { if (isNull) {
colDataSetNull_f(pDst->nullbitmap, 0); colDataSetNull_f(pDst->nullbitmap, 0);
} }
@ -94,11 +101,14 @@ void mJoinTrimKeepFirstRow(SSDataBlock* pBlock) {
} }
pBlock->info.rows = 1; pBlock->info.rows = 1;
return TSDB_CODE_SUCCESS;
} }
void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) { int32_t mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList) {
// int32_t totalRows = pBlock->info.rows; // int32_t totalRows = pBlock->info.rows;
int32_t code = TSDB_CODE_SUCCESS;
int32_t bmLen = BitmapLen(totalRows); int32_t bmLen = BitmapLen(totalRows);
char* pBitmap = NULL; char* pBitmap = NULL;
int32_t maxRows = 0; int32_t maxRows = 0;
@ -106,6 +116,10 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
if (NULL == pDst) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
// it is a reserved column for scalar function, and no data in this column yet. // it is a reserved column for scalar function, and no data in this column yet.
if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) { if (pDst->pData == NULL || (IS_VAR_DATA_TYPE(pDst->info.type) && pDst->varmeta.length == 0)) {
continue; continue;
@ -135,8 +149,12 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
len = varDataTLen(p1); len = varDataTLen(p1);
} }
char* p2 = taosMemoryMalloc(len); char* p2 = taosMemoryMalloc(len);
memcpy(p2, p1, len); TAOS_MEMCPY(p2, p1, len);
colDataSetVal(pDst, numOfRows, p2, false); code = colDataSetVal(pDst, numOfRows, p2, false);
if (code) {
taosMemoryFreeClear(p2);
MJ_ERR_RET(terrno);
}
taosMemoryFree(p2); taosMemoryFree(p2);
} }
numOfRows += 1; numOfRows += 1;
@ -150,10 +168,13 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
} else { } else {
if (pBitmap == NULL) { if (pBitmap == NULL) {
pBitmap = taosMemoryCalloc(1, bmLen); pBitmap = taosMemoryCalloc(1, bmLen);
if (NULL == pBitmap) {
MJ_ERR_RET(terrno);
}
} }
memcpy(pBitmap, pDst->nullbitmap, bmLen); TAOS_MEMCPY(pBitmap, pDst->nullbitmap, bmLen);
memset(pDst->nullbitmap, 0, bmLen); TAOS_MEMSET(pDst->nullbitmap, 0, bmLen);
int32_t j = 0; int32_t j = 0;
@ -243,6 +264,8 @@ void mJoinTrimKeepOneRow(SSDataBlock* pBlock, int32_t totalRows, const bool* pBo
if (pBitmap != NULL) { if (pBitmap != NULL) {
taosMemoryFree(pBitmap); taosMemoryFree(pBitmap);
} }
return TSDB_CODE_SUCCESS;
} }
@ -290,8 +313,10 @@ int32_t mJoinFilterAndMarkHashRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
_err: _err:
colDataDestroy(p); colDataDestroy(p);
taosMemoryFree(p); taosMemoryFree(p);
return code; return code;
} }
@ -300,18 +325,19 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = TSDB_CODE_SUCCESS;
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
SColumnInfoData* p = NULL; SColumnInfoData* p = NULL;
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1); code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _return;
} }
int32_t status = 0; int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status); code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _return;
} }
int32_t rowNum = 0; int32_t rowNum = 0;
@ -320,6 +346,9 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
if (status == FILTER_RESULT_ALL_QUALIFIED || status == FILTER_RESULT_PARTIAL_QUALIFIED) { if (status == FILTER_RESULT_ALL_QUALIFIED || status == FILTER_RESULT_PARTIAL_QUALIFIED) {
for (int32_t i = startGrpIdx; i < grpNum && rowNum < pBlock->info.rows; startRowIdx = 0, ++i) { for (int32_t i = startGrpIdx; i < grpNum && rowNum < pBlock->info.rows; startRowIdx = 0, ++i) {
SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, i); SMJoinGrpRows* buildGrp = taosArrayGet(build->eqGrps, i);
if (NULL == buildGrp) {
MJ_ERR_JRET(terrno);
}
if (buildGrp->allRowsMatch) { if (buildGrp->allRowsMatch) {
rowNum += buildGrp->endIdx - startRowIdx + 1; rowNum += buildGrp->endIdx - startRowIdx + 1;
continue; continue;
@ -350,9 +379,11 @@ int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SM
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
_err: _return:
colDataDestroy(p); colDataDestroy(p);
taosMemoryFree(p); taosMemoryFree(p);
return code; return code;
} }
@ -366,29 +397,31 @@ int32_t mJoinFilterAndKeepSingleRow(SSDataBlock* pBlock, SFilterInfo* pFilterInf
int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1); int32_t code = filterSetDataFromSlotId(pFilterInfo, &param1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _return;
} }
int32_t status = 0; int32_t status = 0;
code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status); code = filterExecute(pFilterInfo, pBlock, &p, NULL, param1.numOfCols, &status);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _return;
} }
if (status == FILTER_RESULT_ALL_QUALIFIED) { if (status == FILTER_RESULT_ALL_QUALIFIED) {
pBlock->info.rows = 1; pBlock->info.rows = 1;
mJoinTrimKeepFirstRow(pBlock); MJ_ERR_JRET(mJoinTrimKeepFirstRow(pBlock));
} else if (status == FILTER_RESULT_NONE_QUALIFIED) { } else if (status == FILTER_RESULT_NONE_QUALIFIED) {
pBlock->info.rows = 0; pBlock->info.rows = 0;
} else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) { } else if (status == FILTER_RESULT_PARTIAL_QUALIFIED) {
mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData); MJ_ERR_JRET(mJoinTrimKeepOneRow(pBlock, pBlock->info.rows, (bool*)p->pData));
} }
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
_err: _return:
colDataDestroy(p); colDataDestroy(p);
taosMemoryFree(p); taosMemoryFree(p);
return code; return code;
} }
@ -418,8 +451,10 @@ int32_t mJoinFilterAndNoKeepRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo)
code = TSDB_CODE_SUCCESS; code = TSDB_CODE_SUCCESS;
_err: _err:
colDataDestroy(p); colDataDestroy(p);
taosMemoryFree(p); taosMemoryFree(p);
return code; return code;
} }
@ -480,12 +515,20 @@ int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pFirstCol = probe->finCols + c; SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pGrp->blk->pDataBlock, pFirstCol->srcSlot); SColumnInfoData* pInCol = taosArrayGet(pGrp->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot);
colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows); if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows, pInCol, pGrp->readIdx, firstRows));
} }
for (int32_t c = 0; c < build->finNum; ++c) { for (int32_t c = 0; c < build->finNum; ++c) {
SMJoinColMap* pSecondCol = build->finCols + c; SMJoinColMap* pSecondCol = build->finCols + c;
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot);
if (NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
colDataSetNItemsNull(pOutCol, currRows, firstRows); colDataSetNItemsNull(pOutCol, currRows, firstRows);
} }
@ -536,6 +579,10 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pFirstCol = probe->finCols + c; SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pFirst->blk->pDataBlock, pFirstCol->srcSlot); SColumnInfoData* pInCol = taosArrayGet(pFirst->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pFirstCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
for (int32_t r = 0; r < firstRows; ++r) { for (int32_t r = 0; r < firstRows; ++r) {
if (colDataIsNull_s(pInCol, pFirst->readIdx + r)) { if (colDataIsNull_s(pInCol, pFirst->readIdx + r)) {
colDataSetNItemsNull(pOutCol, currRows + r * secondRows, secondRows); colDataSetNItemsNull(pOutCol, currRows + r * secondRows, secondRows);
@ -543,7 +590,7 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
ASSERT(pRes->info.capacity >= (pRes->info.rows + firstRows * secondRows)); ASSERT(pRes->info.capacity >= (pRes->info.rows + firstRows * secondRows));
uint32_t startOffset = (IS_VAR_DATA_TYPE(pOutCol->info.type)) ? pOutCol->varmeta.length : ((currRows + r * secondRows) * pOutCol->info.bytes); uint32_t startOffset = (IS_VAR_DATA_TYPE(pOutCol->info.type)) ? pOutCol->varmeta.length : ((currRows + r * secondRows) * pOutCol->info.bytes);
ASSERT((startOffset + 1 * pOutCol->info.bytes) <= pRes->info.capacity * pOutCol->info.bytes); ASSERT((startOffset + 1 * pOutCol->info.bytes) <= pRes->info.capacity * pOutCol->info.bytes);
colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true); MJ_ERR_RET(colDataSetNItems(pOutCol, currRows + r * secondRows, colDataGetData(pInCol, pFirst->readIdx + r), secondRows, true));
} }
} }
} }
@ -552,28 +599,40 @@ int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool app
SMJoinColMap* pSecondCol = build->finCols + c; SMJoinColMap* pSecondCol = build->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(pSecond->blk->pDataBlock, pSecondCol->srcSlot); SColumnInfoData* pInCol = taosArrayGet(pSecond->blk->pDataBlock, pSecondCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot); SColumnInfoData* pOutCol = taosArrayGet(pRes->pDataBlock, pSecondCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
for (int32_t r = 0; r < firstRows; ++r) { for (int32_t r = 0; r < firstRows; ++r) {
colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows); MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r * secondRows, pInCol, pSecond->readIdx, secondRows));
} }
} }
pRes->info.rows = append ? (pRes->info.rows + firstRows * secondRows) : firstRows * secondRows; pRes->info.rows = append ? (pRes->info.rows + firstRows * secondRows) : firstRows * secondRows;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build) { int32_t mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build, bool* cont) {
if (NULL != cont) {
*cont = false;
}
int32_t rowsLeft = append ? (pBlk->info.capacity - pBlk->info.rows) : pBlk->info.capacity; int32_t rowsLeft = append ? (pBlk->info.capacity - pBlk->info.rows) : pBlk->info.capacity;
if (rowsLeft <= 0) { if (rowsLeft <= 0) {
return false; return TSDB_CODE_SUCCESS;
} }
int32_t buildGrpRows = taosArrayGetSize(build->pHashCurGrp); int32_t buildGrpRows = taosArrayGetSize(build->pHashCurGrp);
int32_t grpRows = buildGrpRows - build->grpRowIdx; int32_t grpRows = buildGrpRows - build->grpRowIdx;
if (grpRows <= 0 || build->grpRowIdx < 0) { if (grpRows <= 0 || build->grpRowIdx < 0) {
build->grpRowIdx = -1; build->grpRowIdx = -1;
return true; if (NULL != cont) {
*cont = true;
}
return TSDB_CODE_SUCCESS;
} }
int32_t actRows = TMIN(grpRows, rowsLeft); int32_t actRows = TMIN(grpRows, rowsLeft);
@ -583,10 +642,14 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
SMJoinColMap* pFirstCol = probe->finCols + c; SMJoinColMap* pFirstCol = probe->finCols + c;
SColumnInfoData* pInCol = taosArrayGet(probeGrp->blk->pDataBlock, pFirstCol->srcSlot); SColumnInfoData* pInCol = taosArrayGet(probeGrp->blk->pDataBlock, pFirstCol->srcSlot);
SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pFirstCol->dstSlot); SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pFirstCol->dstSlot);
if (NULL == pInCol || NULL == pOutCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (colDataIsNull_s(pInCol, probeGrp->readIdx)) { if (colDataIsNull_s(pInCol, probeGrp->readIdx)) {
colDataSetNItemsNull(pOutCol, currRows, actRows); colDataSetNItemsNull(pOutCol, currRows, actRows);
} else { } else {
colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true); MJ_ERR_RET(colDataSetNItems(pOutCol, currRows, colDataGetData(pInCol, probeGrp->readIdx), actRows, true));
} }
} }
@ -595,8 +658,16 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pSecondCol->dstSlot); SColumnInfoData* pOutCol = taosArrayGet(pBlk->pDataBlock, pSecondCol->dstSlot);
for (int32_t r = 0; r < actRows; ++r) { for (int32_t r = 0; r < actRows; ++r) {
SMJoinRowPos* pRow = taosArrayGet(build->pHashCurGrp, build->grpRowIdx + r); SMJoinRowPos* pRow = taosArrayGet(build->pHashCurGrp, build->grpRowIdx + r);
if (NULL == pRow) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SColumnInfoData* pInCol = taosArrayGet(pRow->pBlk->pDataBlock, pSecondCol->srcSlot); SColumnInfoData* pInCol = taosArrayGet(pRow->pBlk->pDataBlock, pSecondCol->srcSlot);
colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1); if (NULL == pInCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(colDataAssignNRows(pOutCol, currRows + r, pInCol, pRow->pos, 1));
} }
} }
@ -609,16 +680,24 @@ bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, S
} }
if (actRows == rowsLeft) { if (actRows == rowsLeft) {
return false; return TSDB_CODE_SUCCESS;
} }
return true; if (NULL != cont) {
*cont = true;
}
return TSDB_CODE_SUCCESS;
} }
int32_t mJoinAllocGrpRowBitmap(SMJoinTableCtx* pTb) { int32_t mJoinAllocGrpRowBitmap(SMJoinTableCtx* pTb) {
int32_t grpNum = taosArrayGetSize(pTb->eqGrps); int32_t grpNum = taosArrayGetSize(pTb->eqGrps);
for (int32_t i = 0; i < grpNum; ++i) { for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGet(pTb->eqGrps, i); SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayGet(pTb->eqGrps, i);
if (NULL == pGrp) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
MJ_ERR_RET(mJoinGetRowBitmapOffset(pTb, pGrp->endIdx - pGrp->beginIdx + 1, &pGrp->rowBitmapOffset)); MJ_ERR_RET(mJoinGetRowBitmapOffset(pTb, pGrp->endIdx - pGrp->beginIdx + 1, &pGrp->rowBitmapOffset));
pGrp->rowMatchNum = 0; pGrp->rowMatchNum = 0;
} }
@ -632,9 +711,9 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB
pCtx->lastEqGrp = true; pCtx->lastEqGrp = true;
mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true); MJ_ERR_RET(mJoinBuildEqGroups(pJoin->probe, timestamp, NULL, true));
if (!lastBuildGrp) { if (!lastBuildGrp) {
mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp); MJ_ERR_RET(mJoinRetrieveEqGrpRows(pJoin, pJoin->build, timestamp));
} else { } else {
pJoin->build->grpIdx = 0; pJoin->build->grpIdx = 0;
} }
@ -661,7 +740,7 @@ int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastB
pCtx->hashJoin = false; pCtx->hashJoin = false;
if (!lastBuildGrp && pJoin->build->rowBitmapSize > 0) { if (!lastBuildGrp && pJoin->build->rowBitmapSize > 0) {
mJoinAllocGrpRowBitmap(pJoin->build); MJ_ERR_RET(mJoinAllocGrpRowBitmap(pJoin->build));
} }
return (*pCtx->mergeCartFp)(pCtx); return (*pCtx->mergeCartFp)(pCtx);
@ -721,7 +800,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow
*newDownstreams = true; *newDownstreams = true;
*pDownstream = mJoinBuildDownstreams(pInfo, *pDownstream); *pDownstream = mJoinBuildDownstreams(pInfo, *pDownstream);
if (NULL == *pDownstream) { if (NULL == *pDownstream) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
*numOfDownstream = 2; *numOfDownstream = 2;
} }
@ -732,7 +811,7 @@ int32_t mJoinInitDownstreamInfo(SMJoinOperatorInfo* pInfo, SOperatorInfo*** pDow
static int32_t mJoinInitPrimKeyInfo(SMJoinTableCtx* pTable, int32_t slotId) { static int32_t mJoinInitPrimKeyInfo(SMJoinTableCtx* pTable, int32_t slotId) {
pTable->primCol = taosMemoryMalloc(sizeof(SMJoinColInfo)); pTable->primCol = taosMemoryMalloc(sizeof(SMJoinColInfo));
if (NULL == pTable->primCol) { if (NULL == pTable->primCol) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pTable->primCol->srcSlot = slotId; pTable->primCol->srcSlot = slotId;
@ -745,7 +824,7 @@ static int32_t mJoinInitColsInfo(int32_t* colNum, int64_t* rowSize, SMJoinColInf
*pCols = taosMemoryMalloc((*colNum) * sizeof(SMJoinColInfo)); *pCols = taosMemoryMalloc((*colNum) * sizeof(SMJoinColInfo));
if (NULL == *pCols) { if (NULL == *pCols) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
*rowSize = 0; *rowSize = 0;
@ -779,7 +858,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo
pTable->keyBuf = taosMemoryMalloc(TMAX(rowSize, pTable->keyNullSize)); pTable->keyBuf = taosMemoryMalloc(TMAX(rowSize, pTable->keyNullSize));
if (NULL == pTable->keyBuf) { if (NULL == pTable->keyBuf) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
} }
@ -790,7 +869,7 @@ static int32_t mJoinInitKeyColsInfo(SMJoinTableCtx* pTable, SNodeList* pList, bo
static int32_t mJoinInitFinColsInfo(SMJoinTableCtx* pTable, SNodeList* pList) { static int32_t mJoinInitFinColsInfo(SMJoinTableCtx* pTable, SNodeList* pList) {
pTable->finCols = taosMemoryMalloc(LIST_LENGTH(pList) * sizeof(SMJoinColMap)); pTable->finCols = taosMemoryMalloc(LIST_LENGTH(pList) * sizeof(SMJoinColMap));
if (NULL == pTable->finCols) { if (NULL == pTable->finCols) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
int32_t i = 0; int32_t i = 0;
@ -837,8 +916,20 @@ static int32_t mJoinInitPrimExprCtx(SNode* pNode, SMJoinPrimExprCtx* pCtx, SMJoi
} }
SValueNode* pUnit = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); SValueNode* pUnit = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
SValueNode* pCurrTz = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2) : NULL; if (NULL == pUnit) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
SValueNode* pCurrTz = NULL;
if (5 == pFunc->pParameterList->length){
pCurrTz = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2);
if (NULL == pCurrTz) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
}
SValueNode* pTimeZone = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 4) : (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3); SValueNode* pTimeZone = (5 == pFunc->pParameterList->length) ? (SValueNode*)nodesListGetNode(pFunc->pParameterList, 4) : (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3);
if (NULL == pTimeZone) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
pCtx->truncateUnit = pUnit->typeData; pCtx->truncateUnit = pUnit->typeData;
if ((NULL == pCurrTz || 1 == pCurrTz->typeData) && pCtx->truncateUnit >= (86400 * TSDB_TICK_PER_SECOND(pFunc->node.resType.precision))) { if ((NULL == pCurrTz || 1 == pCurrTz->typeData) && pCtx->truncateUnit >= (86400 * TSDB_TICK_PER_SECOND(pFunc->node.resType.precision))) {
@ -859,24 +950,32 @@ static int32_t mJoinInitTableInfo(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysi
MJ_ERR_RET(mJoinInitKeyColsInfo(pTable, (0 == idx) ? pJoinNode->pEqLeft : pJoinNode->pEqRight, JOIN_TYPE_FULL == pJoin->joinType)); MJ_ERR_RET(mJoinInitKeyColsInfo(pTable, (0 == idx) ? pJoinNode->pEqLeft : pJoinNode->pEqRight, JOIN_TYPE_FULL == pJoin->joinType));
MJ_ERR_RET(mJoinInitFinColsInfo(pTable, pJoinNode->pTargets)); MJ_ERR_RET(mJoinInitFinColsInfo(pTable, pJoinNode->pTargets));
memcpy(&pTable->inputStat, pStat, sizeof(*pStat)); TAOS_MEMCPY(&pTable->inputStat, pStat, sizeof(*pStat));
pTable->eqGrps = taosArrayInit(8, sizeof(SMJoinGrpRows)); pTable->eqGrps = taosArrayInit(8, sizeof(SMJoinGrpRows));
//taosArrayReserve(pTable->eqGrps, 1); if (NULL == pTable->eqGrps) {
return terrno;
}
if (E_JOIN_TB_BUILD == pTable->type) { if (E_JOIN_TB_BUILD == pTable->type) {
pTable->createdBlks = taosArrayInit(8, POINTER_BYTES); pTable->createdBlks = taosArrayInit(8, POINTER_BYTES);
if (NULL == pTable->createdBlks) {
return terrno;
}
pTable->pGrpArrays = taosArrayInit(32, POINTER_BYTES); pTable->pGrpArrays = taosArrayInit(32, POINTER_BYTES);
if (NULL == pTable->pGrpArrays) {
return terrno;
}
pTable->pGrpHash = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); pTable->pGrpHash = tSimpleHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY));
if (NULL == pTable->createdBlks || NULL == pTable->pGrpArrays || NULL == pTable->pGrpHash) { if (NULL == pTable->pGrpHash) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
if (pJoin->pFPreFilter && IS_FULL_OUTER_JOIN(pJoin->joinType, pJoin->subType)) { if (pJoin->pFPreFilter && IS_FULL_OUTER_JOIN(pJoin->joinType, pJoin->subType)) {
pTable->rowBitmapSize = MJOIN_ROW_BITMAP_SIZE; pTable->rowBitmapSize = MJOIN_ROW_BITMAP_SIZE;
pTable->pRowBitmap = taosMemoryMalloc(pTable->rowBitmapSize); pTable->pRowBitmap = taosMemoryMalloc(pTable->rowBitmapSize);
if (NULL == pTable->pRowBitmap) { if (NULL == pTable->pRowBitmap) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
} }
@ -945,7 +1044,15 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SMJoinPrimExprCtx* pCtx = &pTable->primCtx; SMJoinPrimExprCtx* pCtx = &pTable->primCtx;
SColumnInfoData* pPrimIn = taosArrayGet(pBlock->pDataBlock, pTable->primCol->srcSlot); SColumnInfoData* pPrimIn = taosArrayGet(pBlock->pDataBlock, pTable->primCol->srcSlot);
if (NULL == pPrimIn) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId); SColumnInfoData* pPrimOut = taosArrayGet(pBlock->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pPrimOut) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
if (0 != pCtx->timezoneUnit) { if (0 != pCtx->timezoneUnit) {
for (int32_t i = 0; i < pBlock->info.rows; ++i) { for (int32_t i = 0; i < pBlock->info.rows; ++i) {
((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit; ((int64_t*)pPrimOut->pData)[i] = ((int64_t*)pPrimIn->pData)[i] - (((int64_t*)pPrimIn->pData)[i] + pCtx->timezoneUnit) % pCtx->truncateUnit;
@ -961,6 +1068,7 @@ int32_t mJoinLaunchPrimExpr(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable) { SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable) {
SSDataBlock* pTmp = NULL; SSDataBlock* pTmp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
int32_t dsIdx = pTable->downStreamIdx; int32_t dsIdx = pTable->downStreamIdx;
if (E_JOIN_TB_PROBE == pTable->type) { if (E_JOIN_TB_PROBE == pTable->type) {
if (pTable->remainInBlk) { if (pTable->remainInBlk) {
@ -1028,7 +1136,12 @@ SSDataBlock* mJoinGrpRetrieveImpl(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTa
_return: _return:
mJoinLaunchPrimExpr(pTmp, pTable); code = mJoinLaunchPrimExpr(pTmp, pTable);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode);
}
return pTmp; return pTmp;
} }
@ -1041,7 +1154,11 @@ static FORCE_INLINE SSDataBlock* mJoinRetrieveImpl(SMJoinOperatorInfo* pJoin, SM
if (NULL == pTmp) { if (NULL == pTmp) {
pTable->dsFetchDone = true; pTable->dsFetchDone = true;
} else { } else {
mJoinLaunchPrimExpr(pTmp, pTable); int32_t code = mJoinLaunchPrimExpr(pTmp, pTable);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pJoin->pOperator->pTaskInfo->env, pJoin->errCode);
}
} }
return pTmp; return pTmp;
@ -1107,7 +1224,7 @@ bool mJoinRetrieveBlk(SMJoinOperatorInfo* pJoin, int32_t* pIdx, SSDataBlock** pp
static void mJoinDestroyCreatedBlks(SArray* pCreatedBlks) { static void mJoinDestroyCreatedBlks(SArray* pCreatedBlks) {
int32_t blkNum = taosArrayGetSize(pCreatedBlks); int32_t blkNum = taosArrayGetSize(pCreatedBlks);
for (int32_t i = 0; i < blkNum; ++i) { for (int32_t i = 0; i < blkNum; ++i) {
blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i)); (void)blockDataDestroy(*(SSDataBlock**)TARRAY_GET_ELEM(pCreatedBlks, i));
} }
taosArrayClear(pCreatedBlks); taosArrayClear(pCreatedBlks);
} }
@ -1119,12 +1236,12 @@ int32_t mJoinGetRowBitmapOffset(SMJoinTableCtx* pTable, int32_t rowNum, int32_t
int64_t newSize = reqSize * 1.1; int64_t newSize = reqSize * 1.1;
pTable->pRowBitmap = taosMemoryRealloc(pTable->pRowBitmap, newSize); pTable->pRowBitmap = taosMemoryRealloc(pTable->pRowBitmap, newSize);
if (NULL == pTable->pRowBitmap) { if (NULL == pTable->pRowBitmap) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
pTable->rowBitmapSize = newSize; pTable->rowBitmapSize = newSize;
} }
memset(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen); TAOS_MEMSET(pTable->pRowBitmap + pTable->rowBitmapOffset, 0xFFFFFFFF, bitmapLen);
*rowBitmapOffset = pTable->rowBitmapOffset; *rowBitmapOffset = pTable->rowBitmapOffset;
pTable->rowBitmapOffset += bitmapLen; pTable->rowBitmapOffset += bitmapLen;
@ -1140,13 +1257,18 @@ void mJoinResetForBuildTable(SMJoinTableCtx* pTable) {
taosArrayClear(pTable->eqGrps); taosArrayClear(pTable->eqGrps);
if (pTable->rowBitmapSize > 0) { if (pTable->rowBitmapSize > 0) {
pTable->rowBitmapOffset = 1; pTable->rowBitmapOffset = 1;
memset(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx)); TAOS_MEMSET(&pTable->nMatchCtx, 0, sizeof(pTable->nMatchCtx));
} }
} }
int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart) { int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart) {
SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId); SColumnInfoData* pCol = taosArrayGet(pTable->blk->pDataBlock, pTable->primCtx.targetSlotId);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
SMJoinGrpRows* pGrp = NULL; SMJoinGrpRows* pGrp = NULL;
int32_t code = TSDB_CODE_SUCCESS;
if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) { if (*(int64_t*)colDataGetNumData(pCol, pTable->blkRowIdx) != timestamp) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1158,7 +1280,10 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
bool keepGrp = true; bool keepGrp = true;
pGrp = taosArrayReserve(pTable->eqGrps, 1); pGrp = taosArrayReserve(pTable->eqGrps, 1);
if (NULL == pGrp) {
MJ_ERR_RET(terrno);
}
pGrp->beginIdx = pTable->blkRowIdx++; pGrp->beginIdx = pTable->blkRowIdx++;
pGrp->readIdx = pGrp->beginIdx; pGrp->readIdx = pGrp->beginIdx;
pGrp->endIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx;
@ -1207,7 +1332,12 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) { if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) {
pGrp->blk = createOneDataBlock(pTable->blk, true); pGrp->blk = createOneDataBlock(pTable->blk, true);
taosArrayPush(pTable->createdBlks, &pGrp->blk); if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
} else { } else {
if (!pTable->multiEqGrpRows) { if (!pTable->multiEqGrpRows) {
pGrp->endIdx = pGrp->beginIdx; pGrp->endIdx = pGrp->beginIdx;
@ -1230,10 +1360,16 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
pTable->eqRowNum += rowNum; pTable->eqRowNum += rowNum;
pGrp->blk = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum); pGrp->blk = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
}
pGrp->endIdx -= pGrp->beginIdx; pGrp->endIdx -= pGrp->beginIdx;
pGrp->beginIdx = 0; pGrp->beginIdx = 0;
pGrp->readIdx = 0; pGrp->readIdx = 0;
taosArrayPush(pTable->createdBlks, &pGrp->blk); if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
} }
} }
@ -1242,19 +1378,19 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
_return: _return:
if (pTable->noKeepEqGrpRows || !keepGrp || (!pTable->multiEqGrpRows && !restart)) { if (pTable->noKeepEqGrpRows || !keepGrp || (!pTable->multiEqGrpRows && !restart)) {
taosArrayPop(pTable->eqGrps); (void)taosArrayPop(pTable->eqGrps);
} else { } else {
pTable->grpTotalRows += pGrp->endIdx - pGrp->beginIdx + 1; pTable->grpTotalRows += pGrp->endIdx - pGrp->beginIdx + 1;
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp) { int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp) {
bool wholeBlk = false; bool wholeBlk = false;
mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true); MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, true));
while (wholeBlk && !pTable->dsFetchDone) { while (wholeBlk && !pTable->dsFetchDone) {
pTable->blk = (*pJoin->retrieveFp)(pJoin, pTable); pTable->blk = (*pJoin->retrieveFp)(pJoin, pTable);
@ -1267,7 +1403,7 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
} }
wholeBlk = false; wholeBlk = false;
mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false); MJ_ERR_RET(mJoinBuildEqGroups(pTable, timestamp, &wholeBlk, false));
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1276,6 +1412,10 @@ int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable) { int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable) {
for (int32_t i = 0; i < pTable->keyNum; ++i) { for (int32_t i = 0; i < pTable->keyNum; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, pTable->keyCols[i].srcSlot);
if (NULL == pCol) {
MJ_ERR_RET(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
}
if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) { if (pTable->keyCols[i].vardata != IS_VAR_DATA_TYPE(pCol->info.type)) {
qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata); qError("column type mismatch, idx:%d, slotId:%d, type:%d, vardata:%d", i, pTable->keyCols[i].srcSlot, pCol->info.type, pTable->keyCols[i].vardata);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -1320,15 +1460,15 @@ bool mJoinCopyKeyColsDataToBuf(SMJoinTableCtx* pTable, int32_t rowIdx, size_t *p
} }
if (pTable->keyCols[0].jsonData) { if (pTable->keyCols[0].jsonData) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx]; pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData)); TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, getJsonValueLen(pData));
bufLen += getJsonValueLen(pData); bufLen += getJsonValueLen(pData);
} else if (pTable->keyCols[i].vardata) { } else if (pTable->keyCols[i].vardata) {
pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx]; pData = pTable->keyCols[i].data + pTable->keyCols[i].offset[rowIdx];
memcpy(pTable->keyBuf + bufLen, pData, varDataTLen(pData)); TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, varDataTLen(pData));
bufLen += varDataTLen(pData); bufLen += varDataTLen(pData);
} else { } else {
pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx; pData = pTable->keyCols[i].data + pTable->keyCols[i].bytes * rowIdx;
memcpy(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes); TAOS_MEMCPY(pTable->keyBuf + bufLen, pData, pTable->keyCols[i].bytes);
bufLen += pTable->keyCols[i].bytes; bufLen += pTable->keyCols[i].bytes;
} }
} }
@ -1346,15 +1486,20 @@ static int32_t mJoinGetAvailableGrpArray(SMJoinTableCtx* pTable, SArray** ppRes)
do { do {
if (pTable->grpArrayIdx < taosArrayGetSize(pTable->pGrpArrays)) { if (pTable->grpArrayIdx < taosArrayGetSize(pTable->pGrpArrays)) {
*ppRes = taosArrayGetP(pTable->pGrpArrays, pTable->grpArrayIdx++); *ppRes = taosArrayGetP(pTable->pGrpArrays, pTable->grpArrayIdx++);
if (NULL == *ppRes) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
taosArrayClear(*ppRes); taosArrayClear(*ppRes);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SArray* pNew = taosArrayInit(4, sizeof(SMJoinRowPos)); SArray* pNew = taosArrayInit(4, sizeof(SMJoinRowPos));
if (NULL == pNew) { if (NULL == pNew) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
}
if (NULL == taosArrayPush(pTable->pGrpArrays, &pNew)) {
return terrno;
} }
taosArrayPush(pTable->pGrpArrays, &pNew);
} while (true); } while (true);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1368,10 +1513,14 @@ static int32_t mJoinAddRowToHash(SMJoinOperatorInfo* pJoin, size_t keyLen, SSDat
SArray* pNewGrp = NULL; SArray* pNewGrp = NULL;
MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp)); MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp));
taosArrayPush(pNewGrp, &pos); if (NULL == taosArrayPush(pNewGrp, &pos)) {
tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES); return terrno;
}
MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, POINTER_BYTES));
} else if (pBuild->multiRowsGrp) { } else if (pBuild->multiRowsGrp) {
taosArrayPush(*pGrpRows, &pos); if (NULL == taosArrayPush(*pGrpRows, &pos)) {
return terrno;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1386,10 +1535,14 @@ static int32_t mJoinAddRowToFullHash(SMJoinOperatorInfo* pJoin, size_t keyLen, S
SMJoinHashGrpRows pNewGrp = {0}; SMJoinHashGrpRows pNewGrp = {0};
MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp.pRows)); MJ_ERR_RET(mJoinGetAvailableGrpArray(pBuild, &pNewGrp.pRows));
taosArrayPush(pNewGrp.pRows, &pos); if (NULL == taosArrayPush(pNewGrp.pRows, &pos)) {
tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp)); return terrno;
}
MJ_ERR_RET(tSimpleHashPut(pBuild->pGrpHash, pBuild->keyData, keyLen, &pNewGrp, sizeof(pNewGrp)));
} else { } else {
taosArrayPush(pGrpRows->pRows, &pos); if (NULL == taosArrayPush(pGrpRows->pRows, &pos)) {
return terrno;
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1406,6 +1559,9 @@ int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pT
int32_t grpNum = taosArrayGetSize(pTable->eqGrps); int32_t grpNum = taosArrayGetSize(pTable->eqGrps);
for (int32_t g = 0; g < grpNum; ++g) { for (int32_t g = 0; g < grpNum; ++g) {
SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g); SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g);
if (NULL == pGrp) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable)); MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable));
int32_t grpRows = GRP_REMAIN_ROWS(pGrp); int32_t grpRows = GRP_REMAIN_ROWS(pGrp);
@ -1434,6 +1590,10 @@ int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable
int32_t grpNum = taosArrayGetSize(pTable->eqGrps); int32_t grpNum = taosArrayGetSize(pTable->eqGrps);
for (int32_t g = 0; g < grpNum; ++g) { for (int32_t g = 0; g < grpNum; ++g) {
SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g); SMJoinGrpRows* pGrp = taosArrayGet(pTable->eqGrps, g);
if (NULL == pGrp) {
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable)); MJ_ERR_RET(mJoinSetKeyColsData(pGrp->blk, pTable));
int32_t grpRows = GRP_REMAIN_ROWS(pGrp); int32_t grpRows = GRP_REMAIN_ROWS(pGrp);
@ -1490,8 +1650,11 @@ void mWinJoinResetWindowCache(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache) {
for (int32_t i = 0; i < grpNum; ++i) { for (int32_t i = 0; i < grpNum; ++i) {
SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i); SMJoinGrpRows* pGrp = taosArrayGet(pCache->grps, i);
if (NULL == pGrp) {
continue;
}
if (pGrp->blk != pCtx->cache.outBlk && pGrp->clonedBlk) { if (pGrp->blk != pCtx->cache.outBlk && pGrp->clonedBlk) {
blockDataDestroy(pGrp->blk); (void)blockDataDestroy(pGrp->blk);
} }
} }
@ -1533,6 +1696,7 @@ void mJoinResetOperator(struct SOperatorInfo* pOperator) {
SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) { SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
SMJoinOperatorInfo* pJoin = pOperator->info; SMJoinOperatorInfo* pJoin = pOperator->info;
int32_t code = TSDB_CODE_SUCCESS;
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
if (NULL == pOperator->pDownstreamGetParams || NULL == pOperator->pDownstreamGetParams[0] || NULL == pOperator->pDownstreamGetParams[1]) { if (NULL == pOperator->pDownstreamGetParams || NULL == pOperator->pDownstreamGetParams[0] || NULL == pOperator->pDownstreamGetParams[1]) {
qDebug("%s merge join done", GET_TASKID(pOperator->pTaskInfo)); qDebug("%s merge join done", GET_TASKID(pOperator->pTaskInfo));
@ -1553,7 +1717,6 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
pBlock = (*pJoin->joinFp)(pOperator); pBlock = (*pJoin->joinFp)(pOperator);
if (NULL == pBlock) { if (NULL == pBlock) {
if (pJoin->errCode) { if (pJoin->errCode) {
ASSERT(0);
T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode); T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode);
} }
break; break;
@ -1561,7 +1724,11 @@ SSDataBlock* mJoinMainProcess(struct SOperatorInfo* pOperator) {
pBlock->info.id.blockId = pJoin->outBlkId; pBlock->info.id.blockId = pJoin->outBlkId;
if (pJoin->pFinFilter != NULL) { if (pJoin->pFinFilter != NULL) {
doFilter(pBlock, pJoin->pFinFilter, NULL); code = doFilter(pBlock, pJoin->pFinFilter, NULL);
if (code) {
pJoin->errCode = code;
T_LONG_JMP(pOperator->pTaskInfo->env, pJoin->errCode);
}
} }
if (pBlock->info.rows > 0 || pOperator->status == OP_EXEC_DONE) { if (pBlock->info.rows > 0 || pOperator->status == OP_EXEC_DONE) {
@ -1697,13 +1864,17 @@ int32_t mJoinSetImplFp(SMJoinOperatorInfo* pJoin) {
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) {
SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
bool newDownstreams = false; bool newDownstreams = false;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pOperator == NULL || pInfo == NULL) { SMJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMJoinOperatorInfo));
code = TSDB_CODE_OUT_OF_MEMORY; if (pInfo == NULL) {
code = terrno;
goto _return;
}
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL) {
code = terrno;
goto _return; goto _return;
} }
@ -1716,8 +1887,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
MJ_ERR_JRET(mJoinHandleConds(pInfo, pJoinNode)); MJ_ERR_JRET(mJoinHandleConds(pInfo, pJoinNode));
mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams); MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 0, &pJoinNode->inputStat[0], newDownstreams));
mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams); MJ_ERR_JRET(mJoinInitTableInfo(pInfo, pJoinNode, pDownstream, 1, &pJoinNode->inputStat[1], newDownstreams));
MJ_ERR_JRET(mJoinInitCtx(pInfo, pJoinNode)); MJ_ERR_JRET(mJoinInitCtx(pInfo, pJoinNode));
MJ_ERR_JRET(mJoinSetImplFp(pInfo)); MJ_ERR_JRET(mJoinSetImplFp(pInfo));
@ -1736,6 +1907,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
return pOperator; return pOperator;
_return: _return:
if (pInfo != NULL) { if (pInfo != NULL) {
destroyMergeJoinOperator(pInfo); destroyMergeJoinOperator(pInfo);
} }
@ -1745,6 +1917,7 @@ _return:
taosMemoryFree(pOperator); taosMemoryFree(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
return NULL; return NULL;
} }

View File

@ -44,8 +44,8 @@ static int32_t doGenerateSourceData(SOperatorInfo* pOperator);
static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator);
static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator);
static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols); static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols);
static void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, static int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup,
int32_t numOfExprs); int32_t stage, int32_t numOfExprs);
static void destroyProjectOperatorInfo(void* param) { static void destroyProjectOperatorInfo(void* param) {
if (NULL == param) { if (NULL == param) {
@ -142,7 +142,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
} }
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols); code = setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = filterInitFromNode((SNode*)pProjPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pProjPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -447,7 +450,11 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
goto _error; goto _error;
} }
setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); code = setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
code = filterInitFromNode((SNode*)pPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -589,7 +596,8 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
return (rows > 0) ? pInfo->pRes : NULL; return (rows > 0) ? pInfo->pRes : NULL;
} }
void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { int32_t initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t j = 0; j < size; ++j) { for (int32_t j = 0; j < size; ++j) {
struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 ||
@ -597,8 +605,12 @@ void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
continue; continue;
} }
pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo); code = pCtx[j].fpSet.init(&pCtx[j], pCtx[j].resultInfo);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} }
return code;
} }
/* /*
@ -610,7 +622,7 @@ void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) {
* offset[0] offset[1] offset[2] * offset[0] offset[1] offset[2]
*/ */
// TODO refactor: some function move away // TODO refactor: some function move away
void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage,
int32_t numOfExprs) { int32_t numOfExprs) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
@ -632,7 +644,7 @@ void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SA
pCtx[i].scanFlag = stage; pCtx[i].scanFlag = stage;
} }
initCtxOutputBuffer(pCtx, numOfExprs); return initCtxOutputBuffer(pCtx, numOfExprs);
} }
SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) {
@ -841,7 +853,10 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc
// do nothing // do nothing
} else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx);
pfCtx->fpSet.init(pfCtx, pResInfo); code = pfCtx->fpSet.init(pfCtx, pResInfo);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset

View File

@ -202,8 +202,9 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset); SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset);
int32_t reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo); int32_t reqStatus;
if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) { code = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo, &reqStatus);
if (code != TSDB_CODE_SUCCESS || reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) {
notLoadBlock = false; notLoadBlock = false;
break; break;
} }

View File

@ -482,7 +482,8 @@ void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} }
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true); void* pTmpBuf = pBuf;
len = doStreamCountEncodeOpState(&pTmpBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len); strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len);
saveStreamOperatorStateComplete(&pInfo->basic); saveStreamOperatorStateComplete(&pInfo->basic);

View File

@ -542,7 +542,11 @@ void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator); int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamEventEncodeOpState(&pBuf, len, pOperator); len = doStreamEventEncodeOpState(&pBuf, len, pOperator);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);

View File

@ -1410,7 +1410,11 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator); int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator); len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME, pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
@ -3193,7 +3197,11 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true); int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true); len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
@ -4401,7 +4409,11 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true); int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true); len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);

View File

@ -233,6 +233,11 @@ static bool isGroupKeyFunc(SExprInfo* pExprInfo) {
return (functionType == FUNCTION_TYPE_GROUP_KEY); return (functionType == FUNCTION_TYPE_GROUP_KEY);
} }
static bool isSelectGroupConstValueFunc(SExprInfo* pExprInfo) {
int32_t functionType = pExprInfo->pExpr->_function.functionType;
return (functionType == FUNCTION_TYPE_GROUP_CONST_VALUE);
}
static bool getIgoreNullRes(SExprSupp* pExprSup) { static bool getIgoreNullRes(SExprSupp* pExprSup) {
for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[i]; SExprInfo* pExprInfo = &pExprSup->pExprInfo[i];
@ -296,7 +301,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false);
continue; continue;
} else if (!isInterpFunc(pExprInfo)) { } else if (!isInterpFunc(pExprInfo)) {
if (isGroupKeyFunc(pExprInfo)) { if (isGroupKeyFunc(pExprInfo) || isSelectGroupConstValueFunc(pExprInfo)) {
if (pSrcBlock != NULL) { if (pSrcBlock != NULL) {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
@ -308,7 +313,7 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
char* v = colDataGetData(pSrc, index); char* v = colDataGetData(pSrc, index);
colDataSetVal(pDst, pResBlock->info.rows, v, false); colDataSetVal(pDst, pResBlock->info.rows, v, false);
} else { } else if(!isSelectGroupConstValueFunc(pExprInfo)){
// use stored group key // use stored group key
SGroupKeys* pkey = pSliceInfo->pPrevGroupKey; SGroupKeys* pkey = pSliceInfo->pPrevGroupKey;
if (pkey->isNull == false) { if (pkey->isNull == false) {
@ -316,6 +321,14 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
} else { } else {
colDataSetNULL(pDst, rows); colDataSetNULL(pDst, rows);
} }
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
if (pkey->isNull == false) {
colDataSetVal(pDst, rows, pkey->pData, false);
} else {
colDataSetNULL(pDst, rows);
}
} }
} }
continue; continue;

View File

@ -1086,12 +1086,13 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
return (rows == 0) ? NULL : pBlock; return (rows == 0) ? NULL : pBlock;
} }
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { static int32_t doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false);
if (NULL == pResult) { if (NULL == pResult) {
return; return TSDB_CODE_SUCCESS;
} }
int32_t code = TSDB_CODE_SUCCESS;
SqlFunctionCtx* pCtx = pSup->pCtx; SqlFunctionCtx* pCtx = pSup->pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset); pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
@ -1101,15 +1102,19 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf,
} }
pResInfo->initialized = false; pResInfo->initialized = false;
if (pCtx[i].functionId != -1) { if (pCtx[i].functionId != -1) {
pCtx[i].fpSet.init(&pCtx[i], pResInfo); code = pCtx[i].fpSet.init(&pCtx[i], pResInfo);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} }
} }
SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId); SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
if (NULL == bufPage) { if (NULL == bufPage) {
return; return TSDB_CODE_SUCCESS;
} }
setBufPageDirty(bufPage, true); setBufPageDirty(bufPage, true);
releaseBufPage(pResultBuf, bufPage); releaseBufPage(pResultBuf, bufPage);
return TSDB_CODE_SUCCESS;
} }
static void destroyStateWindowOperatorInfo(void* param) { static void destroyStateWindowOperatorInfo(void* param) {

File diff suppressed because it is too large Load Diff

View File

@ -49,9 +49,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems)
int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); int32_t loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos, char** value);
bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult); int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
@ -74,7 +74,7 @@ int32_t sumInvertFunction(SqlFunctionCtx* pCtx);
int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t minFunction(SqlFunctionCtx* pCtx); int32_t minFunction(SqlFunctionCtx* pCtx);
int32_t maxFunction(SqlFunctionCtx* pCtx); int32_t maxFunction(SqlFunctionCtx* pCtx);
@ -83,7 +83,7 @@ int32_t minCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t avgFunction(SqlFunctionCtx* pCtx); int32_t avgFunction(SqlFunctionCtx* pCtx);
int32_t avgFunctionMerge(SqlFunctionCtx* pCtx); int32_t avgFunctionMerge(SqlFunctionCtx* pCtx);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -97,7 +97,7 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getAvgInfoSize(); int32_t getAvgInfoSize();
bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFunction(SqlFunctionCtx* pCtx);
int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx); int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -111,18 +111,18 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getStddevInfoSize(); int32_t getStddevInfoSize();
bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t leastSQRFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t leastSQRFunction(SqlFunctionCtx* pCtx); int32_t leastSQRFunction(SqlFunctionCtx* pCtx);
int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t leastSQRFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t leastSQRCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t percentileFunction(SqlFunctionCtx* pCtx); int32_t percentileFunction(SqlFunctionCtx* pCtx);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t apercentileFunction(SqlFunctionCtx* pCtx); int32_t apercentileFunction(SqlFunctionCtx* pCtx);
int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx); int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -131,16 +131,16 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
int32_t getApercentileMaxSize(); int32_t getApercentileMaxSize();
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t diffFunction(SqlFunctionCtx* pCtx); int32_t diffFunction(SqlFunctionCtx* pCtx);
int32_t diffFunctionByRow(SArray* pCtx); int32_t diffFunctionByRow(SArray* pCtx);
bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t derivativeFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t derivativeFunction(SqlFunctionCtx* pCtx); int32_t derivativeFunction(SqlFunctionCtx* pCtx);
bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo); int32_t irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo);
int32_t irateFunction(SqlFunctionCtx* pCtx); int32_t irateFunction(SqlFunctionCtx* pCtx);
int32_t irateFunctionMerge(SqlFunctionCtx* pCtx); int32_t irateFunctionMerge(SqlFunctionCtx* pCtx);
int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -165,7 +165,7 @@ EFuncDataRequired lastDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo);
int32_t lastRowFunction(SqlFunctionCtx* pCtx); int32_t lastRowFunction(SqlFunctionCtx* pCtx);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t topFunction(SqlFunctionCtx* pCtx); int32_t topFunction(SqlFunctionCtx* pCtx);
int32_t bottomFunction(SqlFunctionCtx* pCtx); int32_t bottomFunction(SqlFunctionCtx* pCtx);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -174,7 +174,7 @@ int32_t bottomCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t getTopBotInfoSize(int64_t numOfItems); int32_t getTopBotInfoSize(int64_t numOfItems);
bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t spreadFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t spreadFunction(SqlFunctionCtx* pCtx); int32_t spreadFunction(SqlFunctionCtx* pCtx);
int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx); int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -183,7 +183,7 @@ int32_t getSpreadInfoSize();
int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t spreadCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t elapsedFunction(SqlFunctionCtx* pCtx); int32_t elapsedFunction(SqlFunctionCtx* pCtx);
int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx); int32_t elapsedFunctionMerge(SqlFunctionCtx* pCtx);
int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -192,7 +192,7 @@ int32_t getElapsedInfoSize();
int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t elapsedCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t histogramFunction(SqlFunctionCtx* pCtx); int32_t histogramFunction(SqlFunctionCtx* pCtx);
int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx); int32_t histogramFunctionPartial(SqlFunctionCtx* pCtx);
int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx); int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx);
@ -210,7 +210,7 @@ int32_t getHLLInfoSize();
int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t hllCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getStateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stateFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t stateFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stateCountFunction(SqlFunctionCtx* pCtx); int32_t stateCountFunction(SqlFunctionCtx* pCtx);
int32_t stateDurationFunction(SqlFunctionCtx* pCtx); int32_t stateDurationFunction(SqlFunctionCtx* pCtx);
@ -218,35 +218,35 @@ bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t csumFunction(SqlFunctionCtx* pCtx); int32_t csumFunction(SqlFunctionCtx* pCtx);
bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getMavgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t mavgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t mavgFunction(SqlFunctionCtx* pCtx); int32_t mavgFunction(SqlFunctionCtx* pCtx);
bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSampleFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t sampleFunction(SqlFunctionCtx* pCtx); int32_t sampleFunction(SqlFunctionCtx* pCtx);
int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getTailFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t tailFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t tailFunction(SqlFunctionCtx* pCtx); int32_t tailFunction(SqlFunctionCtx* pCtx);
bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t uniqueFunction(SqlFunctionCtx* pCtx); int32_t uniqueFunction(SqlFunctionCtx* pCtx);
bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t modeFunction(SqlFunctionCtx* pCtx); int32_t modeFunction(SqlFunctionCtx* pCtx);
int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t twaFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t twaFunction(SqlFunctionCtx* pCtx); int32_t twaFunction(SqlFunctionCtx* pCtx);
int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo); int32_t blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo);
int32_t blockDistFunction(SqlFunctionCtx* pCtx); int32_t blockDistFunction(SqlFunctionCtx* pCtx);
int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
@ -254,6 +254,8 @@ bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t groupKeyFunction(SqlFunctionCtx* pCtx); int32_t groupKeyFunction(SqlFunctionCtx* pCtx);
int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
int32_t groupConstValueFunction(SqlFunctionCtx* pCtx);
int32_t groupConstValueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -37,7 +37,7 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32
pResInfo->complete = false; pResInfo->complete = false;
pResInfo->numOfRes = 0; pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); (void)memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
} }
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -55,14 +55,15 @@ typedef struct SHistogramInfo {
#endif #endif
} SHistogramInfo; } SHistogramInfo;
SHistogramInfo* tHistogramCreate(int32_t numOfBins); int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto);
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins); SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins);
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val); int32_t tHistogramAdd(SHistogramInfo** pHisto, double val);
int64_t tHistogramSum(SHistogramInfo* pHisto, double v); int64_t tHistogramSum(SHistogramInfo* pHisto, double v);
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num); int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal);
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries); int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries,
SHistogramInfo** pResHistogram);
void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramDestroy(SHistogramInfo** pHisto);
void tHistogramPrint(SHistogramInfo* pHisto); void tHistogramPrint(SHistogramInfo* pHisto);

View File

@ -67,7 +67,8 @@ typedef struct tMemBucket {
SHashObj *groupPagesMap; // disk page map for different groups; SHashObj *groupPagesMap; // disk page map for different groups;
} tMemBucket; } tMemBucket;
tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup); int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup,
tMemBucket **pBucket);
void tMemBucketDestroy(tMemBucket *pBucket); void tMemBucketDestroy(tMemBucket *pBucket);

View File

@ -25,7 +25,7 @@
static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) { static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) {
va_list vArgList; va_list vArgList;
va_start(vArgList, pFormat); va_start(vArgList, pFormat);
vsnprintf(pErrBuf, len, pFormat, vArgList); (void)vsnprintf(pErrBuf, len, pFormat, vArgList);
va_end(vArgList); va_end(vArgList);
return errCode; return errCode;
} }
@ -42,27 +42,24 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName); return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
} }
#define TIME_UNIT_INVALID 1
#define TIME_UNIT_TOO_SMALL 2
static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) { static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
if (!IS_DURATION_VAL(pVal->flag)) { if (!IS_DURATION_VAL(pVal->flag)) {
return TIME_UNIT_INVALID; return TSDB_CODE_FUNC_TIME_UNIT_INVALID;
} }
if (TSDB_TIME_PRECISION_MILLI == dbPrec && if (TSDB_TIME_PRECISION_MILLI == dbPrec &&
(0 == strcasecmp(pVal->literal, "1u") || 0 == strcasecmp(pVal->literal, "1b"))) { (0 == strcasecmp(pVal->literal, "1u") || 0 == strcasecmp(pVal->literal, "1b"))) {
return TIME_UNIT_TOO_SMALL; return TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL;
} }
if (TSDB_TIME_PRECISION_MICRO == dbPrec && 0 == strcasecmp(pVal->literal, "1b")) { if (TSDB_TIME_PRECISION_MICRO == dbPrec && 0 == strcasecmp(pVal->literal, "1b")) {
return TIME_UNIT_TOO_SMALL; return TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL;
} }
if (pVal->literal[0] != '1' || if (pVal->literal[0] != '1' ||
(pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && pVal->literal[1] != 's' && pVal->literal[1] != 'm' && (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' && pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) { pVal->literal[1] != 'h' && pVal->literal[1] != 'd' && pVal->literal[1] != 'w' && pVal->literal[1] != 'b')) {
return TIME_UNIT_INVALID; return TSDB_CODE_FUNC_TIME_UNIT_INVALID;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -138,13 +135,13 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
} }
if (i == 2) { if (i == 2) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
hour = taosStr2Int8(buf, NULL, 10); hour = taosStr2Int8(buf, NULL, 10);
if (!validateHourRange(hour)) { if (!validateHourRange(hour)) {
return false; return false;
} }
} else if (i == 4) { } else if (i == 4) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
minute = taosStr2Int8(buf, NULL, 10); minute = taosStr2Int8(buf, NULL, 10);
if (!validateMinuteRange(hour, minute, tz[0])) { if (!validateMinuteRange(hour, minute, tz[0])) {
return false; return false;
@ -167,13 +164,13 @@ static bool validateTimezoneFormat(const SValueNode* pVal) {
} }
if (i == 2) { if (i == 2) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
hour = taosStr2Int8(buf, NULL, 10); hour = taosStr2Int8(buf, NULL, 10);
if (!validateHourRange(hour)) { if (!validateHourRange(hour)) {
return false; return false;
} }
} else if (i == 5) { } else if (i == 5) {
memcpy(buf, &tz[i - 1], 2); (void)memcpy(buf, &tz[i - 1], 2);
minute = taosStr2Int8(buf, NULL, 10); minute = taosStr2Int8(buf, NULL, 10);
if (!validateMinuteRange(hour, minute, tz[0])) { if (!validateMinuteRange(hour, minute, tz[0])) {
return false; return false;
@ -215,7 +212,7 @@ static int32_t addTimezoneParam(SNodeList* pList) {
time_t t = taosTime(NULL); time_t t = taosTime(NULL);
struct tm tmInfo; struct tm tmInfo;
if (taosLocalTime(&t, &tmInfo, buf) != NULL) { if (taosLocalTime(&t, &tmInfo, buf) != NULL) {
strftime(buf, sizeof(buf), "%z", &tmInfo); (void)strftime(buf, sizeof(buf), "%z", &tmInfo);
} }
int32_t len = (int32_t)strlen(buf); int32_t len = (int32_t)strlen(buf);
@ -225,15 +222,27 @@ static int32_t addTimezoneParam(SNodeList* pList) {
} }
pVal->literal = strndup(buf, len); pVal->literal = strndup(buf, len);
if (pVal->literal == NULL) {
nodesDestroyNode((SNode*)pVal);
return TSDB_CODE_OUT_OF_MEMORY;
}
pVal->translate = true; pVal->translate = true;
pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE; pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI; pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
if (pVal->datum.p == NULL) {
nodesDestroyNode((SNode*)pVal);
return TSDB_CODE_OUT_OF_MEMORY;
}
varDataSetLen(pVal->datum.p, len); varDataSetLen(pVal->datum.p, len);
strncpy(varDataVal(pVal->datum.p), pVal->literal, len); (void)strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
nodesListAppend(pList, (SNode*)pVal); int32_t code = nodesListAppend(pList, (SNode*)pVal);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pVal);
return code;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -252,7 +261,11 @@ static int32_t addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
pVal->datum.i = (int64_t)precision; pVal->datum.i = (int64_t)precision;
pVal->typeData = (int64_t)precision; pVal->typeData = (int64_t)precision;
nodesListMakeAppend(pList, (SNode*)pVal); int32_t code = nodesListMakeAppend(pList, (SNode*)pVal);
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pVal);
return code;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -936,13 +949,13 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"ELAPSED function time unit parameter should be greater than db precision"); "ELAPSED function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg( return buildFuncErrMsg(
pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, pErrBuf, len, code,
"ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); "ELAPSED function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
} }
} }
@ -1062,7 +1075,7 @@ static int8_t validateHistogramBinType(char* binTypeStr) {
return binType; return binType;
} }
static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) { static int32_t validateHistogramBinDesc(char* binDescStr, int8_t binType, char* errMsg, int32_t msgLen) {
const char* msg1 = "HISTOGRAM function requires four parameters"; const char* msg1 = "HISTOGRAM function requires four parameters";
const char* msg3 = "HISTOGRAM function invalid format for binDesc parameter"; const char* msg3 = "HISTOGRAM function invalid format for binDesc parameter";
const char* msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]"; const char* msg4 = "HISTOGRAM function binDesc parameter \"count\" should be in range [1, 1000]";
@ -1070,6 +1083,7 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
const char* msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0"; const char* msg6 = "HISTOGRAM function binDesc parameter \"width\" cannot be 0";
const char* msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type"; const char* msg7 = "HISTOGRAM function binDesc parameter \"start\" cannot be 0 with \"log_bin\" type";
const char* msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1"; const char* msg8 = "HISTOGRAM function binDesc parameter \"factor\" cannot be negative or equal to 0/1";
const char* msg9 = "HISTOGRAM function out of memory";
cJSON* binDesc = cJSON_Parse(binDescStr); cJSON* binDesc = cJSON_Parse(binDescStr);
int32_t numOfBins; int32_t numOfBins;
@ -1078,9 +1092,9 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
int32_t numOfParams = cJSON_GetArraySize(binDesc); int32_t numOfParams = cJSON_GetArraySize(binDesc);
int32_t startIndex; int32_t startIndex;
if (numOfParams != 4) { if (numOfParams != 4) {
snprintf(errMsg, msgLen, "%s", msg1); (void)snprintf(errMsg, msgLen, "%s", msg1);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
cJSON* start = cJSON_GetObjectItem(binDesc, "start"); cJSON* start = cJSON_GetObjectItem(binDesc, "start");
@ -1090,22 +1104,22 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity");
if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000
snprintf(errMsg, msgLen, "%s", msg4); (void)snprintf(errMsg, msgLen, "%s", msg4);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) ||
(factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) {
snprintf(errMsg, msgLen, "%s", msg5); (void)snprintf(errMsg, msgLen, "%s", msg5);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
int32_t counter = (int32_t)count->valueint; int32_t counter = (int32_t)count->valueint;
@ -1118,53 +1132,58 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
} }
intervals = taosMemoryCalloc(numOfBins, sizeof(double)); intervals = taosMemoryCalloc(numOfBins, sizeof(double));
if (intervals == NULL) {
(void)snprintf(errMsg, msgLen, "%s", msg9);
cJSON_Delete(binDesc);
return TSDB_CODE_FAILED;
}
if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) {
// linear bin process // linear bin process
if (width->valuedouble == 0) { if (width->valuedouble == 0) {
snprintf(errMsg, msgLen, "%s", msg6); (void)snprintf(errMsg, msgLen, "%s", msg6);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
for (int i = 0; i < counter + 1; ++i) { for (int i = 0; i < counter + 1; ++i) {
intervals[startIndex] = start->valuedouble + i * width->valuedouble; intervals[startIndex] = start->valuedouble + i * width->valuedouble;
if (isinf(intervals[startIndex])) { if (isinf(intervals[startIndex])) {
snprintf(errMsg, msgLen, "%s", msg5); (void)snprintf(errMsg, msgLen, "%s", msg5);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
startIndex++; startIndex++;
} }
} else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) {
// log bin process // log bin process
if (start->valuedouble == 0) { if (start->valuedouble == 0) {
snprintf(errMsg, msgLen, "%s", msg7); (void)snprintf(errMsg, msgLen, "%s", msg7);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) {
snprintf(errMsg, msgLen, "%s", msg8); (void)snprintf(errMsg, msgLen, "%s", msg8);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
for (int i = 0; i < counter + 1; ++i) { for (int i = 0; i < counter + 1; ++i) {
intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0);
if (isinf(intervals[startIndex])) { if (isinf(intervals[startIndex])) {
snprintf(errMsg, msgLen, "%s", msg5); (void)snprintf(errMsg, msgLen, "%s", msg5);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
startIndex++; startIndex++;
} }
} else { } else {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (infinity->valueint == true) { if (infinity->valueint == true) {
@ -1172,7 +1191,7 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
intervals[numOfBins - 1] = INFINITY; intervals[numOfBins - 1] = INFINITY;
// in case of desc bin orders, -inf/inf should be swapped // in case of desc bin orders, -inf/inf should be swapped
if (numOfBins < 4) { if (numOfBins < 4) {
return false; return TSDB_CODE_FAILED;
} }
if (intervals[1] > intervals[numOfBins - 2]) { if (intervals[1] > intervals[numOfBins - 2]) {
@ -1181,46 +1200,51 @@ static bool validateHistogramBinDesc(char* binDescStr, int8_t binType, char* err
} }
} else if (cJSON_IsArray(binDesc)) { /* user input bins */ } else if (cJSON_IsArray(binDesc)) { /* user input bins */
if (binType != USER_INPUT_BIN) { if (binType != USER_INPUT_BIN) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
numOfBins = cJSON_GetArraySize(binDesc); numOfBins = cJSON_GetArraySize(binDesc);
intervals = taosMemoryCalloc(numOfBins, sizeof(double)); intervals = taosMemoryCalloc(numOfBins, sizeof(double));
if (intervals == NULL) {
(void)snprintf(errMsg, msgLen, "%s", msg9);
cJSON_Delete(binDesc);
return TSDB_CODE_OUT_OF_MEMORY;
}
cJSON* bin = binDesc->child; cJSON* bin = binDesc->child;
if (bin == NULL) { if (bin == NULL) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
int i = 0; int i = 0;
while (bin) { while (bin) {
intervals[i] = bin->valuedouble; intervals[i] = bin->valuedouble;
if (!cJSON_IsNumber(bin)) { if (!cJSON_IsNumber(bin)) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
if (i != 0 && intervals[i] <= intervals[i - 1]) { if (i != 0 && intervals[i] <= intervals[i - 1]) {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
taosMemoryFree(intervals); taosMemoryFree(intervals);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
bin = bin->next; bin = bin->next;
i++; i++;
} }
} else { } else {
snprintf(errMsg, msgLen, "%s", msg3); (void)snprintf(errMsg, msgLen, "%s", msg3);
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
return false; return TSDB_CODE_FAILED;
} }
cJSON_Delete(binDesc); cJSON_Delete(binDesc);
taosMemoryFree(intervals); taosMemoryFree(intervals);
return true; return TSDB_CODE_SUCCESS;
} }
static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
@ -1265,7 +1289,7 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
if (i == 2) { if (i == 2) {
char errMsg[128] = {0}; char errMsg[128] = {0};
binDesc = varDataVal(pValue->datum.p); binDesc = varDataVal(pValue->datum.p);
if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { if (TSDB_CODE_SUCCESS != validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg); return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg);
} }
} }
@ -1323,7 +1347,7 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
if (i == 2) { if (i == 2) {
char errMsg[128] = {0}; char errMsg[128] = {0};
binDesc = varDataVal(pValue->datum.p); binDesc = varDataVal(pValue->datum.p);
if (!validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) { if (TSDB_CODE_SUCCESS != validateHistogramBinDesc(binDesc, binType, errMsg, (int32_t)sizeof(errMsg))) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg); return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, errMsg);
} }
} }
@ -1507,12 +1531,12 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
if (numOfParams == 4) { if (numOfParams == 4) {
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 3));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"STATEDURATION function time unit parameter should be greater than db precision"); "STATEDURATION function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, " "STATEDURATION function time unit parameter should be one of the following: [1b, 1u, 1a, "
"1s, 1m, 1h, 1d, 1w]"); "1s, 1m, 1h, 1d, 1w]");
} }
@ -2268,13 +2292,13 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
} }
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"TIMETRUNCATE function time unit parameter should be greater than db precision"); "TIMETRUNCATE function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg( return buildFuncErrMsg(
pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, pErrBuf, len, code,
"TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
} }
@ -2291,7 +2315,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
// add database precision as param // add database precision as param
int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -2330,13 +2354,13 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
uint8_t dbPrec = pFunc->node.resType.precision; uint8_t dbPrec = pFunc->node.resType.precision;
if (3 == numOfParams) { if (3 == numOfParams) {
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2)); int32_t code = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2));
if (ret == TIME_UNIT_TOO_SMALL) { if (code == TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL) {
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, return buildFuncErrMsg(pErrBuf, len, code,
"TIMEDIFF function time unit parameter should be greater than db precision"); "TIMEDIFF function time unit parameter should be greater than db precision");
} else if (ret == TIME_UNIT_INVALID) { } else if (code == TSDB_CODE_FUNC_TIME_UNIT_INVALID) {
return buildFuncErrMsg( return buildFuncErrMsg(
pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, pErrBuf, len, code,
"TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); "TIMEDIFF function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
} }
} }
@ -4108,6 +4132,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = md5Function, .sprocessFunc = md5Function,
.finalizeFunc = NULL .finalizeFunc = NULL
}, },
{
.name = "_group_const_value",
.type = FUNCTION_TYPE_GROUP_CONST_VALUE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateSelectValue,
.getEnvFunc = getSelectivityFuncEnv,
.initFunc = functionSetup,
.processFunc = groupConstValueFunction,
.finalizeFunc = groupConstValueFinalize,
},
}; };
// clang-format on // clang-format on

File diff suppressed because it is too large Load Diff

View File

@ -354,14 +354,17 @@ bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true; return true;
} }
bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { int32_t avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) { if (pResultInfo->initialized) {
return false; return TSDB_CODE_SUCCESS;
}
if (TSDB_CODE_SUCCESS != functionSetup(pCtx, pResultInfo)) {
return TSDB_CODE_FUNC_SETUP_ERROR;
} }
SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
memset(pRes, 0, sizeof(SAvgRes)); (void)memset(pRes, 0, sizeof(SAvgRes));
return true; return TSDB_CODE_SUCCESS;
} }
static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) { static int32_t calculateAvgBySMAInfo(SAvgRes* pRes, int32_t numOfRows, int32_t type, const SColumnDataAgg* pAgg) {
@ -849,15 +852,23 @@ int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
int32_t resultBytes = getAvgInfoSize(); int32_t resultBytes = getAvgInfoSize();
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
int32_t code = TSDB_CODE_SUCCESS;
memcpy(varDataVal(res), pInfo, resultBytes); if (NULL == res) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(varDataVal(res), pInfo, resultBytes);
varDataSetLen(res, resultBytes); varDataSetLen(res, resultBytes);
int32_t slotId = pCtx->pExpr->base.resSchema.slotId; int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
if(NULL == pCol) {
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
colDataSetVal(pCol, pBlock->info.rows, res, false); code = colDataSetVal(pCol, pBlock->info.rows, res, false);
_exit:
taosMemoryFree(res); taosMemoryFree(res);
return pResInfo->numOfRes; return code;
} }

View File

@ -20,6 +20,7 @@
#include "tglobal.h" #include "tglobal.h"
#define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \ #define __COMPARE_ACQUIRED_MAX(i, end, bm, _data, ctx, val, pos) \
int32_t code = TSDB_CODE_SUCCESS; \
for (; i < (end); ++i) { \ for (; i < (end); ++i) { \
if (colDataIsNull_f(bm, i)) { \ if (colDataIsNull_f(bm, i)) { \
continue; \ continue; \
@ -28,12 +29,16 @@
if ((val) < (_data)[i]) { \ if ((val) < (_data)[i]) { \
(val) = (_data)[i]; \ (val) = (_data)[i]; \
if ((ctx)->subsidiaries.num > 0) { \ if ((ctx)->subsidiaries.num > 0) { \
updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ code = updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \
if (TSDB_CODE_SUCCESS != code) { \
return code; \
} \
} \ } \
} \ } \
} }
#define __COMPARE_ACQUIRED_MIN(i, end, bm, _data, ctx, val, pos) \ #define __COMPARE_ACQUIRED_MIN(i, end, bm, _data, ctx, val, pos) \
int32_t code = TSDB_CODE_SUCCESS; \
for (; i < (end); ++i) { \ for (; i < (end); ++i) { \
if (colDataIsNull_f(bm, i)) { \ if (colDataIsNull_f(bm, i)) { \
continue; \ continue; \
@ -42,7 +47,10 @@
if ((val) > (_data)[i]) { \ if ((val) > (_data)[i]) { \
(val) = (_data)[i]; \ (val) = (_data)[i]; \
if ((ctx)->subsidiaries.num > 0) { \ if ((ctx)->subsidiaries.num > 0) { \
updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \ code = updateTupleData((ctx), i, (ctx)->pSrcBlock, pos); \
if (TSDB_CODE_SUCCESS != code) { \
return code; \
} \
} \ } \
} \ } \
} }
@ -571,7 +579,7 @@ static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, c
return -1; return -1;
} }
static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf, static int32_t doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunctionCtx* pCtx, SMinmaxResInfo* pBuf,
bool isMinFunc) { bool isMinFunc) {
if (isMinFunc) { if (isMinFunc) {
switch (pCol->info.type) { switch (pCol->info.type) {
@ -700,6 +708,7 @@ static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunct
} }
} }
} }
return TSDB_CODE_SUCCESS;
} }
static int32_t saveRelatedTupleTag(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, void* tval) { static int32_t saveRelatedTupleTag(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, void* tval) {
@ -840,7 +849,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems)
break; break;
} }
default: default:
memcpy(&pBuf->v, p, pCol->info.bytes); (void)memcpy(&pBuf->v, p, pCol->info.bytes);
break; break;
} }
@ -858,7 +867,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems)
goto _over; goto _over;
} }
doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc); code = doExtractVal(pCol, i, end, pCtx, pBuf, isMinFunc);
} else { } else {
numOfElems = numOfRows; numOfElems = numOfRows;

View File

@ -35,14 +35,14 @@ static void doInitFunctionTable() {
gFunMgtService.pFuncNameHashTable = gFunMgtService.pFuncNameHashTable =
taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); taosHashInit(funcMgtBuiltinsNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (NULL == gFunMgtService.pFuncNameHashTable) { if (NULL == gFunMgtService.pFuncNameHashTable) {
initFunctionCode = TSDB_CODE_FAILED; initFunctionCode = terrno;
return; return;
} }
for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) { for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) {
if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name, if (TSDB_CODE_SUCCESS != taosHashPut(gFunMgtService.pFuncNameHashTable, funcMgtBuiltins[i].name,
strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) { strlen(funcMgtBuiltins[i].name), &i, sizeof(int32_t))) {
initFunctionCode = TSDB_CODE_FAILED; initFunctionCode = terrno;
return; return;
} }
} }
@ -61,7 +61,7 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
} }
int32_t fmFuncMgtInit() { int32_t fmFuncMgtInit() {
taosThreadOnce(&functionHashTableInit, doInitFunctionTable); (void)taosThreadOnce(&functionHashTableInit, doInitFunctionTable);
return initFunctionCode; return initFunctionCode;
} }
@ -115,20 +115,24 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow); return funcMgtBuiltins[pFunc->funcId].dataRequiredFunc(pFunc, pTimeWindow);
} }
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo) { int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus) {
if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) {
*reqStatus = -1;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
const char* name = funcMgtBuiltins[funcId].name; const char* name = funcMgtBuiltins[funcId].name;
if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) { if ((strcmp(name, "_group_key") == 0) || (strcmp(name, "_select_value") == 0)) {
return FUNC_DATA_REQUIRED_NOT_LOAD; *reqStatus = FUNC_DATA_REQUIRED_NOT_LOAD;
return TSDB_CODE_SUCCESS;;
} }
if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) { if (funcMgtBuiltins[funcId].dynDataRequiredFunc == NULL) {
return FUNC_DATA_REQUIRED_DATA_LOAD; *reqStatus = FUNC_DATA_REQUIRED_DATA_LOAD;
return TSDB_CODE_SUCCESS;
} else { } else {
return funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo); *reqStatus = funcMgtBuiltins[funcId].dynDataRequiredFunc(pRes, pBlockInfo);
return TSDB_CODE_SUCCESS;
} }
} }
@ -268,6 +272,13 @@ bool fmIsGroupKeyFunc(int32_t funcId) {
return FUNCTION_TYPE_GROUP_KEY == funcMgtBuiltins[funcId].type; return FUNCTION_TYPE_GROUP_KEY == funcMgtBuiltins[funcId].type;
} }
bool fmisSelectGroupConstValueFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false;
}
return FUNCTION_TYPE_GROUP_CONST_VALUE == funcMgtBuiltins[funcId].type;
}
bool fmIsBlockDistFunc(int32_t funcId) { bool fmIsBlockDistFunc(int32_t funcId) {
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
return false; return false;
@ -371,29 +382,30 @@ static int32_t getFuncInfo(SFunctionNode* pFunc) {
return fmGetFuncInfo(pFunc, msg, sizeof(msg)); return fmGetFuncInfo(pFunc, msg, sizeof(msg));
} }
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); *pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) { if (NULL == *pFunc) {
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pName); (void)snprintf((*pFunc)->functionName, sizeof((*pFunc)->functionName), "%s", pName);
pFunc->pParameterList = pParameterList; (*pFunc)->pParameterList = pParameterList;
if (TSDB_CODE_SUCCESS != getFuncInfo(pFunc)) { int32_t code = getFuncInfo((*pFunc));
pFunc->pParameterList = NULL; if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pFunc); (*pFunc)->pParameterList = NULL;
return NULL; nodesDestroyNode((SNode*)*pFunc);
return code;
} }
return pFunc; return code;
} }
static SNode* createColumnByFunc(const SFunctionNode* pFunc) { static int32_t createColumnByFunc(const SFunctionNode* pFunc, SColumnNode** pCol) {
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); *pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) { if (NULL == *pCol) {
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
strcpy(pCol->colName, pFunc->node.aliasName); (void)strcpy((*pCol)->colName, pFunc->node.aliasName);
pCol->node.resType = pFunc->node.resType; (*pCol)->node.resType = pFunc->node.resType;
return (SNode*)pCol; return TSDB_CODE_SUCCESS;
} }
bool fmIsDistExecFunc(int32_t funcId) { bool fmIsDistExecFunc(int32_t funcId) {
@ -411,17 +423,17 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
if (NULL == pParameterList) { if (NULL == pParameterList) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
*pPartialFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList); int32_t code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList,pPartialFunc );
if (NULL == *pPartialFunc) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pParameterList); nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
(*pPartialFunc)->hasOriginalFunc = true; (*pPartialFunc)->hasOriginalFunc = true;
(*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId; (*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0}; char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0};
int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc); int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
taosCreateMD5Hash(name, len); (void)taosCreateMD5Hash(name, len);
strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1); (void)strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1);
(*pPartialFunc)->hasPk = pSrcFunc->hasPk; (*pPartialFunc)->hasPk = pSrcFunc->hasPk;
(*pPartialFunc)->pkBytes = pSrcFunc->pkBytes; (*pPartialFunc)->pkBytes = pSrcFunc->pkBytes;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -429,7 +441,11 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc, static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc,
SNodeList** pParameterList) { SNodeList** pParameterList) {
SNode* pRes = createColumnByFunc(pPartialFunc); SNode *pRes = NULL;
int32_t code = createColumnByFunc(pPartialFunc, (SColumnNode**)&pRes);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL != funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc) { if (NULL != funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc) {
return funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc(pSrcFunc->pParameterList, pRes, pParameterList); return funcMgtBuiltins[pSrcFunc->funcId].createMergeParaFuc(pSrcFunc->pParameterList, pRes, pParameterList);
} else { } else {
@ -445,16 +461,13 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN
int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){ if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList); code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList, &pFunc);
}else{ }else{
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc);
}
if (NULL == pFunc) {
code = TSDB_CODE_OUT_OF_MEMORY;
} }
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName); (void)strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -474,10 +487,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList); int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); code = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList, &pFunc);
if (NULL == pFunc) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pFunc->hasOriginalFunc = true; pFunc->hasOriginalFunc = true;
@ -486,7 +496,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
if (fmIsSameInOutType(pSrcFunc->funcId)) { if (fmIsSameInOutType(pSrcFunc->funcId)) {
pFunc->node.resType = pSrcFunc->node.resType; pFunc->node.resType = pSrcFunc->node.resType;
} }
strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName); (void)strcpy(pFunc->node.aliasName, pSrcFunc->node.aliasName);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -534,13 +544,13 @@ static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pSt
if (funcMgtBuiltins[pFunc->funcId].pStateFunc) { if (funcMgtBuiltins[pFunc->funcId].pStateFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList); SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY; if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams); int32_t code = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams, pStateFunc);
if (!*pStateFunc) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pParams); nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR; return TSDB_CODE_FUNC_FUNTION_ERROR;
} }
strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName); (void)strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias); (void)strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -580,13 +590,13 @@ static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pSta
if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) { if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList); SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY; if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateMergeFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams); int32_t code = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams, pStateMergeFunc);
if (!*pStateMergeFunc) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyList(pParams); nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR; return code;
} }
strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName); (void)strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias); (void)strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -639,6 +649,9 @@ bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) {
} }
if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true; if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true;
int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc); int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc);
if (stateMergeFuncId == -1) {
return false;
}
const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId]; const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId];
return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0; return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0;
} }

View File

@ -32,9 +32,12 @@
*/ */
static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val); static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
SHistogramInfo* tHistogramCreate(int32_t numOfEntries) { int32_t tHistogramCreate(int32_t numOfEntries, SHistogramInfo** pHisto) {
/* need one redundant slot */ /* need one redundant slot */
SHistogramInfo* pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1)); *pHisto = taosMemoryMalloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1));
if (NULL == *pHisto) {
return TSDB_CODE_OUT_OF_MEMORY;
}
#if !defined(USE_ARRAYLIST) #if !defined(USE_ARRAYLIST)
pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
@ -46,11 +49,12 @@ SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
pss->pTree = pHisto->pLoserTree; pss->pTree = pHisto->pLoserTree;
#endif #endif
return tHistogramCreateFrom(pHisto, numOfEntries); *pHisto = tHistogramCreateFrom(*pHisto, numOfEntries);
return TSDB_CODE_SUCCESS;
} }
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) { SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1)); (void)memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1));
SHistogramInfo* pHisto = (SHistogramInfo*)pBuf; SHistogramInfo* pHisto = (SHistogramInfo*)pBuf;
pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo)); pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo));
@ -67,15 +71,19 @@ SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
} }
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
int32_t code = TSDB_CODE_SUCCESS;
if (*pHisto == NULL) { if (*pHisto == NULL) {
*pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN); code = tHistogramCreate(MAX_HISTOGRAM_BIN, pHisto);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
} }
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p", if (ASSERTS(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL, "tHistogramAdd Error, idx:%d, maxEntries:%d, elems:%p",
idx, (*pHisto)->maxEntries, (*pHisto)->elems)) { idx, (*pHisto)->maxEntries, (*pHisto)->elems)) {
return -1; return TSDB_CODE_FAILED;
} }
if ((*pHisto)->elems[idx].val == val && idx >= 0) { if ((*pHisto)->elems[idx].val == val && idx >= 0) {
@ -89,23 +97,23 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
if (idx > 0) { if (idx > 0) {
if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", if (ASSERTS((*pHisto)->elems[idx - 1].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf",
idx - 1, (*pHisto)->elems[idx - 1].val, val)) { idx - 1, (*pHisto)->elems[idx - 1].val, val)) {
return -1; return TSDB_CODE_FAILED;
} }
} else { } else {
if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", if (ASSERTS((*pHisto)->elems[idx].val > val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf",
idx, (*pHisto)->elems[idx].val, val)) { idx, (*pHisto)->elems[idx].val, val)) {
return -1; return TSDB_CODE_FAILED;
} }
} }
} else if ((*pHisto)->numOfElems > 0) { } else if ((*pHisto)->numOfElems > 0) {
if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf", if (ASSERTS((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val, "tHistogramAdd Error, elems[%d].val:%lf, val:%lf",
(*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) { (*pHisto)->numOfEntries, (*pHisto)->elems[idx].val, val)) {
return -1; return TSDB_CODE_FAILED;
} }
} }
int32_t code = histogramCreateBin(*pHisto, idx, val); code = histogramCreateBin(*pHisto, idx, val);
if (code != 0) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
} }
@ -286,7 +294,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
} }
(*pHisto)->numOfElems += 1; (*pHisto)->numOfElems += 1;
return 0; return code;
} }
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
@ -335,7 +343,7 @@ static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) {
s1->val = newVal; s1->val = newVal;
s1->num = s1->num + s2->num; s1->num = s1->num + s2->num;
memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin)); (void)memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin));
(*size) -= 1; (*size) -= 1;
#endif #endif
} }
@ -345,12 +353,12 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
int32_t remain = pHisto->numOfEntries - index; int32_t remain = pHisto->numOfEntries - index;
if (remain > 0) { if (remain > 0) {
memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain); (void)memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain);
} }
if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d", if (ASSERTS(index >= 0 && index <= pHisto->maxEntries, "histogramCreateBin Error, index:%d, maxEntries:%d",
index, pHisto->maxEntries)) { index, pHisto->maxEntries)) {
return -1; return TSDB_CODE_FAILED;
} }
pHisto->elems[index].num = 1; pHisto->elems[index].num = 1;
@ -367,10 +375,10 @@ int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
#endif #endif
if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d", if (ASSERTS(pHisto->numOfEntries <= pHisto->maxEntries, "histogramCreateBin Error, numOfEntries:%d, maxEntries:%d",
pHisto->numOfEntries, pHisto->maxEntries)) { pHisto->numOfEntries, pHisto->maxEntries)) {
return -1; return TSDB_CODE_FAILED;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
void tHistogramDestroy(SHistogramInfo** pHisto) { void tHistogramDestroy(SHistogramInfo** pHisto) {
@ -383,17 +391,17 @@ void tHistogramDestroy(SHistogramInfo** pHisto) {
} }
void tHistogramPrint(SHistogramInfo* pHisto) { void tHistogramPrint(SHistogramInfo* pHisto) {
printf("total entries: %d, elements: %" PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems); (void)printf("total entries: %d, elements: %" PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems);
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num); (void)printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
} }
#else #else
tSkipListNode* pNode = pHisto->pList->pHead.pForward[0]; tSkipListNode* pNode = pHisto->pList->pHead.pForward[0];
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) { for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
SHistBin* pEntry = (SHistBin*)pNode->pData; SHistBin* pEntry = (SHistBin*)pNode->pData;
printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num); (void)printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num);
pNode = pNode->pForward[0]; pNode = pNode->pForward[0];
} }
#endif #endif
@ -443,21 +451,24 @@ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
#endif #endif
} }
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) { int32_t tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num, double** pVal) {
#if defined(USE_ARRAYLIST) #if defined(USE_ARRAYLIST)
double* pVal = taosMemoryMalloc(num * sizeof(double)); *pVal = taosMemoryMalloc(num * sizeof(double));
if (NULL == *pVal) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
double numOfElem = (ratio[i] / 100) * pHisto->numOfElems; double numOfElem = (ratio[i] / 100) * pHisto->numOfElems;
if (numOfElem == 0) { if (numOfElem == 0) {
pVal[i] = pHisto->min; (*pVal)[i] = pHisto->min;
continue; continue;
} else if (numOfElem <= pHisto->elems[0].num) { } else if (numOfElem <= pHisto->elems[0].num) {
pVal[i] = pHisto->elems[0].val; (*pVal)[i] = pHisto->elems[0].val;
continue; continue;
} else if (numOfElem == pHisto->numOfElems) { } else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max; (*pVal)[i] = pHisto->max;
continue; continue;
} }
@ -479,37 +490,39 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
double delta = numOfElem - total; double delta = numOfElem - total;
if (fabs(delta) < FLT_EPSILON) { if (fabs(delta) < FLT_EPSILON) {
pVal[i] = pHisto->elems[j].val; (*pVal)[i] = pHisto->elems[j].val;
} }
double start = (double)pHisto->elems[j].num; double start = (double)pHisto->elems[j].num;
double range = pHisto->elems[j + 1].num - start; double range = pHisto->elems[j + 1].num - start;
if (range == 0) { if (range == 0) {
pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val; (*pVal)[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val;
} else { } else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor; (*pVal)[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor;
} }
} }
#else #else
double* pVal = taosMemoryMalloc(num * sizeof(double)); double* pVal = taosMemoryMalloc(num * sizeof(double));
if (NULL == *pVal) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
double numOfElem = ratio[i] * pHisto->numOfElems; double numOfElem = ratio[i] * pHisto->numOfElems;
tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0]; tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0];
SHistBin* pEntry = (SHistBin*)pFirst->pData; SHistBin* pEntry = (SHistBin*)pFirst->pData;
if (numOfElem == 0) { if (numOfElem == 0) {
pVal[i] = pHisto->min; (*pVal)[i] = pHisto->min;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue; continue;
} else if (numOfElem <= pEntry->num) { } else if (numOfElem <= pEntry->num) {
pVal[i] = pEntry->val; (*pVal)[i] = pEntry->val;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue; continue;
} else if (numOfElem == pHisto->numOfElems) { } else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max; (*pVal)[i] = pHisto->max;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]); printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue; continue;
} }
@ -540,34 +553,40 @@ double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
if (fabs(delta) < FLT_EPSILON) { if (fabs(delta) < FLT_EPSILON) {
// printf("i/numofSlot: %f, v:%f, %f\n", // printf("i/numofSlot: %f, v:%f, %f\n",
// (double)i/numOfSlots, numOfElem, pHisto->elems[j].val); // (double)i/numOfSlots, numOfElem, pHisto->elems[j].val);
pVal[i] = pPrev->val; (*pVal)[i] = pPrev->val;
} }
double start = pPrev->num; double start = pPrev->num;
double range = pEntry->num - start; double range = pEntry->num - start;
if (range == 0) { if (range == 0) {
pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val; (*pVal)[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val;
} else { } else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range); double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor; (*pVal)[i] = pPrev->val + (pEntry->val - pPrev->val) * factor;
} }
// printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots, // printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots,
// numOfElem, val); // numOfElem, val);
} }
#endif #endif
return pVal; return TSDB_CODE_SUCCESS;
} }
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) { int32_t tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries, SHistogramInfo** pResHistogram) {
SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries); int32_t code = tHistogramCreate(numOfEntries, pResHistogram);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
// error in histogram info // error in histogram info
if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) { if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) {
return pResHistogram; return code;
} }
SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries)); SHistBin* pHistoBins = taosMemoryCalloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries));
if (NULL == pHistoBins) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t i = 0, j = 0, k = 0; int32_t i = 0, j = 0, k = 0;
while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) { while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) {
@ -583,28 +602,28 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2
if (i < pHisto1->numOfEntries) { if (i < pHisto1->numOfEntries) {
int32_t remain = pHisto1->numOfEntries - i; int32_t remain = pHisto1->numOfEntries - i;
memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain); (void)memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain);
k += remain; k += remain;
} }
if (j < pHisto2->numOfEntries) { if (j < pHisto2->numOfEntries) {
int32_t remain = pHisto2->numOfEntries - j; int32_t remain = pHisto2->numOfEntries - j;
memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain); (void)memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain);
k += remain; k += remain;
} }
/* update other information */ /* update other information */
pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems; (*pResHistogram)->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems;
pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min; (*pResHistogram)->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min;
pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max; (*pResHistogram)->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max;
while (k > numOfEntries) { while (k > numOfEntries) {
histogramMergeImpl(pHistoBins, &k); histogramMergeImpl(pHistoBins, &k);
} }
pResHistogram->numOfEntries = k; (*pResHistogram)->numOfEntries = k;
memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k); (void)memcpy((*pResHistogram)->elems, pHistoBins, sizeof(SHistBin) * k);
taosMemoryFree(pHistoBins); taosMemoryFree(pHistoBins);
return pResHistogram; return TSDB_CODE_SUCCESS;
} }

View File

@ -28,9 +28,12 @@
int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { return (times * numOfSlots) + slotIndex; } int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) { return (times * numOfSlots) + slotIndex; }
static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) { static int32_t loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx, SFilePage ** buffer) {
SFilePage *buffer = *buffer =
(SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage));
if (NULL == *buffer) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times);
@ -39,26 +42,30 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
if (p != NULL) { if (p != NULL) {
pIdList = *(SArray **)p; pIdList = *(SArray **)p;
} else { } else {
taosMemoryFree(buffer); taosMemoryFree(*buffer);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t offset = 0; int32_t offset = 0;
for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) {
int32_t *pageId = taosArrayGet(pIdList, i); int32_t *pageId = taosArrayGet(pIdList, i);
if (pageId == NULL) {
taosMemoryFree(*buffer);
return TSDB_CODE_OUT_OF_RANGE;
}
SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId);
if (pg == NULL) { if (pg == NULL) {
taosMemoryFree(buffer); taosMemoryFree(*buffer);
return NULL; return terrno;
} }
memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); (void)memcpy((*buffer)->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
offset += (int32_t)(pg->num * pMemBucket->bytes); offset += (int32_t)(pg->num * pMemBucket->bytes);
} }
taosSort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); taosSort((*buffer)->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
return buffer; return TSDB_CODE_SUCCESS;
} }
static void resetBoundingBox(MinMaxEntry *range, int32_t type) { static void resetBoundingBox(MinMaxEntry *range, int32_t type) {
@ -116,6 +123,9 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) {
ASSERT(list->size == 1); ASSERT(list->size == 1);
int32_t *pageId = taosArrayGet(list, 0); int32_t *pageId = taosArrayGet(list, 0);
if (NULL == pageId) {
return TSDB_CODE_OUT_OF_RANGE;
}
SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId); SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId);
if (pPage == NULL) { if (pPage == NULL) {
return terrno; return terrno;
@ -238,67 +248,71 @@ static void resetSlotInfo(tMemBucket *pBucket) {
} }
} }
tMemBucket *tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup) { int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, double maxval, bool hasWindowOrGroup,
tMemBucket *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket)); tMemBucket **pBucket) {
if (pBucket == NULL) { *pBucket = (tMemBucket *)taosMemoryCalloc(1, sizeof(tMemBucket));
return NULL; if (*pBucket == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
if (hasWindowOrGroup) { if (hasWindowOrGroup) {
// With window or group by, we need to shrink page size and reduce page num to save memory. // With window or group by, we need to shrink page size and reduce page num to save memory.
pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket (*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT / 8 ; // 128 bucket
pBucket->bufPageSize = 4096; // 4k per page (*pBucket)->bufPageSize = 4096; // 4k per page
} else { } else {
pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; (*pBucket)->numOfSlots = DEFAULT_NUM_OF_SLOT;
pBucket->bufPageSize = 16384 * 4; // 16k per page (*pBucket)->bufPageSize = 16384 * 4; // 16k per page
} }
pBucket->type = dataType; (*pBucket)->type = dataType;
pBucket->bytes = nElemSize; (*pBucket)->bytes = nElemSize;
pBucket->total = 0; (*pBucket)->total = 0;
pBucket->times = 1; (*pBucket)->times = 1;
pBucket->maxCapacity = 200000; (*pBucket)->maxCapacity = 200000;
pBucket->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); (*pBucket)->groupPagesMap = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) { if ((*pBucket)->groupPagesMap == NULL) {
tMemBucketDestroy(*pBucket);
return terrno;
}
if (setBoundingBox(&(*pBucket)->range, (*pBucket)->type, minval, maxval) != 0) {
// qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval); // qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
taosMemoryFree(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_FUNC_INVALID_VALUE_RANGE;
} }
pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(SFilePage)) / pBucket->bytes; (*pBucket)->elemPerPage = ((*pBucket)->bufPageSize - sizeof(SFilePage)) / (*pBucket)->bytes;
pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC); (*pBucket)->comparFn = getKeyComparFunc((*pBucket)->type, TSDB_ORDER_ASC);
pBucket->hashFunc = getHashFunc(pBucket->type); (*pBucket)->hashFunc = getHashFunc((*pBucket)->type);
if (pBucket->hashFunc == NULL) { if ((*pBucket)->hashFunc == NULL) {
// qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type); // qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type);
taosMemoryFree(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
} }
pBucket->pSlots = (tMemBucketSlot *)taosMemoryCalloc(pBucket->numOfSlots, sizeof(tMemBucketSlot)); (*pBucket)->pSlots = (tMemBucketSlot *)taosMemoryCalloc((*pBucket)->numOfSlots, sizeof(tMemBucketSlot));
if (pBucket->pSlots == NULL) { if ((*pBucket)->pSlots == NULL) {
taosMemoryFree(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
resetSlotInfo(pBucket); resetSlotInfo((*pBucket));
if (!osTempSpaceAvailable()) { if (!osTempSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE;
// qError("MemBucket create disk based Buf failed since %s", terrstr(terrno)); // qError("MemBucket create disk based Buf failed since %s", terrstr(terrno));
tMemBucketDestroy(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return TSDB_CODE_NO_DISKSPACE;
} }
int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 1024, "1", tsTempDir); int32_t ret = createDiskbasedBuf(&(*pBucket)->pBuffer, (*pBucket)->bufPageSize, (*pBucket)->bufPageSize * 1024, "1", tsTempDir);
if (ret != 0) { if (ret != 0) {
tMemBucketDestroy(pBucket); tMemBucketDestroy(*pBucket);
return NULL; return ret;
} }
// qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes); // qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes);
return pBucket; return TSDB_CODE_SUCCESS;
} }
void tMemBucketDestroy(tMemBucket *pBucket) { void tMemBucketDestroy(tMemBucket *pBucket) {
@ -394,7 +408,14 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId));
if (p == NULL) { if (p == NULL) {
pPageIdList = taosArrayInit(4, sizeof(int32_t)); pPageIdList = taosArrayInit(4, sizeof(int32_t));
taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); if (NULL == pPageIdList) {
return terrno;
}
int32_t code = taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES);
if (TSDB_CODE_SUCCESS != code) {
taosArrayDestroy(pPageIdList);
return code;
}
} else { } else {
pPageIdList = *(SArray **)p; pPageIdList = *(SArray **)p;
} }
@ -404,10 +425,13 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
return terrno; return terrno;
} }
pSlot->info.pageId = pageId; pSlot->info.pageId = pageId;
taosArrayPush(pPageIdList, &pageId); if (taosArrayPush(pPageIdList, &pageId) == NULL) {
taosArrayDestroy(pPageIdList);
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes); (void)memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes);
pSlot->info.data->num += 1; pSlot->info.data->num += 1;
pSlot->info.size += 1; pSlot->info.size += 1;
@ -493,9 +517,10 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
if (pSlot->info.size <= pMemBucket->maxCapacity) { if (pSlot->info.size <= pMemBucket->maxCapacity) {
// data in buffer and file are merged together to be processed. // data in buffer and file are merged together to be processed.
SFilePage *buffer = loadDataFromFilePage(pMemBucket, i); SFilePage *buffer = NULL;
if (buffer == NULL) { int32_t code = loadDataFromFilePage(pMemBucket, i, &buffer);
return terrno; if (TSDB_CODE_SUCCESS != code) {
return code;
} }
int32_t currentIdx = count - num; int32_t currentIdx = count - num;
@ -541,6 +566,9 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
for (int32_t f = 0; f < list->size; ++f) { for (int32_t f = 0; f < list->size; ++f) {
int32_t *pageId = taosArrayGet(list, f); int32_t *pageId = taosArrayGet(list, f);
if (NULL == pageId) {
return TSDB_CODE_OUT_OF_RANGE;
}
SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId);
if (pg == NULL) { if (pg == NULL) {
return terrno; return terrno;

View File

@ -1011,7 +1011,7 @@ void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle);
int32_t cleanUpUdfs(); int32_t cleanUpUdfs();
bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
@ -1196,15 +1196,18 @@ bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv) {
return true; return true;
} }
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) { int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo) {
if (functionSetup(pCtx, pResultCellInfo) != true) { if (pResultCellInfo->initialized) {
return false; return TSDB_CODE_SUCCESS;
}
if (functionSetup(pCtx, pResultCellInfo) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FUNC_SETUP_ERROR;
} }
UdfcFuncHandle handle; UdfcFuncHandle handle;
int32_t udfCode = 0; int32_t udfCode = 0;
if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) { if ((udfCode = acquireUdfFuncHandle((char *)pCtx->udfName, &handle)) != 0) {
fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode); fnError("udfAggInit error. step doSetupUdf. udf code: %d", udfCode);
return false; return TSDB_CODE_FUNC_SETUP_ERROR;
} }
SUdfcUvSession *session = (SUdfcUvSession *)handle; SUdfcUvSession *session = (SUdfcUvSession *)handle;
SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(pResultCellInfo); SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(pResultCellInfo);
@ -1218,7 +1221,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult
if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) {
fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode);
releaseUdfFuncHandle(pCtx->udfName, handle); releaseUdfFuncHandle(pCtx->udfName, handle);
return false; return TSDB_CODE_FUNC_SETUP_ERROR;
} }
if (buf.bufLen <= session->bufSize) { if (buf.bufLen <= session->bufSize) {
memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); memcpy(udfRes->interResBuf, buf.buf, buf.bufLen);
@ -1227,11 +1230,11 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult
} else { } else {
fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize);
releaseUdfFuncHandle(pCtx->udfName, handle); releaseUdfFuncHandle(pCtx->udfName, handle);
return false; return TSDB_CODE_FUNC_SETUP_ERROR;
} }
releaseUdfFuncHandle(pCtx->udfName, handle); releaseUdfFuncHandle(pCtx->udfName, handle);
freeUdfInterBuf(&buf); freeUdfInterBuf(&buf);
return true; return TSDB_CODE_SUCCESS;
} }
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {

View File

@ -180,7 +180,7 @@ int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnI
goto _exit; goto _exit;
} }
colDataSetVal(pOutputData, i, output, (output == NULL)); code = colDataSetVal(pOutputData, i, output, (output == NULL));
_exit: _exit:
if (output) { if (output) {
@ -200,7 +200,7 @@ int32_t executeAsTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoDat
goto _exit; goto _exit;
} }
colDataSetVal(pOutputData, i, output, (output == NULL)); code = colDataSetVal(pOutputData, i, output, (output == NULL));
_exit: _exit:
if (output) { if (output) {
@ -228,7 +228,7 @@ int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometr
} }
} }
colDataSetVal(pOutputData, i, &res, (res==-1)); code = colDataSetVal(pOutputData, i, &res, (res==-1));
return code; return code;
} }

View File

@ -86,9 +86,9 @@ _exit:
return code; return code;
} }
static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) { static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
int ret = 0; int32_t code = 0;
char *wktPatternWithSpace = taosMemoryCalloc(4, 1024); char *wktPatternWithSpace = taosMemoryCalloc(4, 1024);
sprintf( sprintf(
wktPatternWithSpace, wktPatternWithSpace,
"^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
@ -142,9 +142,9 @@ static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( "
"*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$"); "*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$");
ret = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace); code = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace);
taosMemoryFree(wktPatternWithSpace); taosMemoryFree(wktPatternWithSpace);
return ret; return code;
} }
int32_t initCtxGeomFromText() { int32_t initCtxGeomFromText() {

View File

@ -1523,7 +1523,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod
if (!multiResFunc) { if (!multiResFunc) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN);
code = nodesListMakeStrictAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); code = nodesListMakeStrictAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc);
@ -3162,6 +3162,25 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
} }
static EDealRes rewriteExprToSelectTagFunc(STranslateContext* pCxt, SNode** pNode) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
strcpy(pFunc->functionName, "_group_const_value");
strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName);
strcpy(pFunc->node.userAlias, ((SExprNode*)*pNode)->userAlias);
pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode);
if (TSDB_CODE_SUCCESS == pCxt->errCode) {
*pNode = (SNode*)pFunc;
pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
}
return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
}
static bool isWindowJoinProbeTablePrimCol(SSelectStmt* pSelect, SNode* pNode) { static bool isWindowJoinProbeTablePrimCol(SSelectStmt* pSelect, SNode* pNode) {
if (QUERY_NODE_COLUMN != nodeType(pNode)) { if (QUERY_NODE_COLUMN != nodeType(pNode)) {
return false; return false;
@ -3388,13 +3407,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
if (nodesEqualNode(pActualNode, *pNode)) { if (nodesEqualNode(pActualNode, *pNode)) {
return DEAL_RES_IGNORE_CHILD; return DEAL_RES_IGNORE_CHILD;
} }
if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) &&
((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) {
return rewriteExprToGroupKeyFunc(pCxt, pNode);
}
if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) {
return rewriteExprToGroupKeyFunc(pCxt, pNode); return rewriteExprToGroupKeyFunc(pCxt, pNode);
} }
if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) &&
((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) {
return rewriteExprToSelectTagFunc(pCxt, pNode);
}
} }
SNode* pPartKey = NULL; SNode* pPartKey = NULL;
bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList);
@ -3494,10 +3513,13 @@ static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) {
} }
} }
if (partionByTbname && if (partionByTbname &&
((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) || (QUERY_NODE_FUNCTION == nodeType(*pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)*pNode)->funcType)) {
(QUERY_NODE_FUNCTION == nodeType(*pNode) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)*pNode)->funcType))) {
return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode);
} }
if (partionByTbname &&
((QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG))) {
return rewriteExprToSelectTagFunc(pCxt->pTranslateCxt, pNode);
}
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
pCxt->existCol = true; pCxt->existCol = true;
} }
@ -4951,7 +4973,8 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) {
} }
static EDealRes needFillImpl(SNode* pNode, void* pContext) { static EDealRes needFillImpl(SNode* pNode, void* pContext) {
if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { if ((isAggFunc(pNode) || isInterpFunc(pNode)) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType
&& FUNCTION_TYPE_GROUP_CONST_VALUE != ((SFunctionNode*)pNode)->funcType) {
*(bool*)pContext = true; *(bool*)pContext = true;
return DEAL_RES_END; return DEAL_RES_END;
} }
@ -5210,7 +5233,8 @@ static int32_t translateFill(STranslateContext* pCxt, SSelectStmt* pSelect, SInt
} }
static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) {
int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); int64_t days = -1;
convertTimeFromPrecisionToUnit(val, fromPrecision, 'd', &days);
switch (unit) { switch (unit) {
case 'b': case 'b':
case 'u': case 'u':
@ -8209,7 +8233,8 @@ static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) {
if (NULL == pVal) { if (NULL == pVal) {
return NULL; return NULL;
} }
int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit); int64_t timeVal = -1;
convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit, &timeVal);
char buf[20] = {0}; char buf[20] = {0};
int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit);
pVal->literal = strndup(buf, len); pVal->literal = strndup(buf, len);
@ -8758,7 +8783,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tNameExtractFullName(&name, usedbReq.db); code = tNameExtractFullName(&name, usedbReq.db);
} }
if (TSDB_CODE_SUCCESS == code) if (TSDB_CODE_SUCCESS == code)
code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq);
@ -10395,8 +10420,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char*
return code; return code;
} }
SNode* pFunc = (SNode*)createFunction("last", pParameterList); SNode* pFunc = NULL;
if (NULL == pFunc) { code = createFunction("last", pParameterList, (SFunctionNode**)&pFunc);
if (code) {
nodesDestroyList(pParameterList); nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -10413,8 +10439,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char*
return code; return code;
} }
SFunctionNode* pFunc1 = createFunction("_vgid", NULL); SFunctionNode* pFunc1 = NULL;
if (NULL == pFunc1) { code = createFunction("_vgid", NULL, &pFunc1);
if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -10426,8 +10453,9 @@ static int32_t createLastTsSelectStmt(char* pDb, const char* pTable, const char*
return code; return code;
} }
SFunctionNode* pFunc2 = createFunction("_vgver", NULL); SFunctionNode* pFunc2 = NULL;
if (NULL == pFunc2) { code = createFunction("_vgver", NULL, &pFunc2);
if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }

View File

@ -35,7 +35,8 @@ namespace ParserTest {
class ParserEnv : public testing::Environment { class ParserEnv : public testing::Environment {
public: public:
virtual void SetUp() { virtual void SetUp() {
fmFuncMgtInit(); // TODO(smj) : How to handle return value of fmFuncMgtInit
(void)fmFuncMgtInit();
initMetaDataEnv(); initMetaDataEnv();
generateMetaData(); generateMetaData();
initLog(TD_TMP_DIR_PATH "td"); initLog(TD_TMP_DIR_PATH "td");

View File

@ -891,7 +891,7 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
} }
static bool isInterpFunc(int32_t funcId) { static bool isInterpFunc(int32_t funcId) {
return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId); return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId) || fmisSelectGroupConstValueFunc(funcId);
} }
static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {

View File

@ -5336,9 +5336,13 @@ static bool stbJoinOptShouldBeOptimized(SLogicNode* pNode) {
} }
int32_t stbJoinOptAddFuncToScanNode(char* funcName, SScanLogicNode* pScan) { int32_t stbJoinOptAddFuncToScanNode(char* funcName, SScanLogicNode* pScan) {
SFunctionNode* pUidFunc = createFunction(funcName, NULL); SFunctionNode* pUidFunc = NULL;
int32_t code = createFunction(funcName, NULL, &pUidFunc);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
snprintf(pUidFunc->node.aliasName, sizeof(pUidFunc->node.aliasName), "%s.%p", pUidFunc->functionName, pUidFunc); snprintf(pUidFunc->node.aliasName, sizeof(pUidFunc->node.aliasName), "%s.%p", pUidFunc->functionName, pUidFunc);
int32_t code = nodesListStrictAppend(pScan->pScanPseudoCols, (SNode*)pUidFunc); code = nodesListStrictAppend(pScan->pScanPseudoCols, (SNode*)pUidFunc);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createColumnByRewriteExpr((SNode*)pUidFunc, &pScan->node.pTargets); code = createColumnByRewriteExpr((SNode*)pUidFunc, &pScan->node.pTargets);
} }

View File

@ -27,7 +27,8 @@
class PlannerEnv : public testing::Environment { class PlannerEnv : public testing::Environment {
public: public:
virtual void SetUp() { virtual void SetUp() {
fmFuncMgtInit(); // TODO(smj) : How to handle return value of fmFuncMgtInit
(void)fmFuncMgtInit();
initMetaDataEnv(); initMetaDataEnv();
generateMetaData(); generateMetaData();
initLog(TD_TMP_DIR_PATH "td"); initLog(TD_TMP_DIR_PATH "td");

Some files were not shown because too many files have changed in this diff Show More