Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/TD-30987-13

This commit is contained in:
Hongze Cheng 2024-07-23 15:02:33 +08:00
commit 391a31c5a2
39 changed files with 3110 additions and 2055 deletions

View File

@ -86,11 +86,11 @@ void deltaToUtcInitOnce();
char getPrecisionUnit(int32_t precision); char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit); int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision); int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
struct STm { struct STm {
struct tm tm; struct tm tm;

View File

@ -686,7 +686,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
// checkpoint related // checkpoint related
int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId); void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId);
int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId); int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId);
int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId); int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId);
bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId); bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId);
@ -770,9 +770,9 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask); int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer // timer
tmr_h streamTimerGetInstance(); int32_t streamTimerGetInstance(tmr_h* pTmr);
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg); const char* pMsg);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
@ -809,6 +809,9 @@ void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp); int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask); int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -2491,7 +2491,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
int32_t code = buildChildTableName(&rname); int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
taosArrayDestroy(tags); taosArrayDestroy(tags);

View File

@ -150,7 +150,6 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
cap -= nwrite; cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) { for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) { if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else { } else {

View File

@ -35,7 +35,7 @@ void deltaToUtcInitOnce() {
// printf("====delta:%lld\n\n", seconds); // printf("====delta:%lld\n\n", seconds);
} }
static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction);
static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
@ -95,7 +95,9 @@ char* forwardToTimeStringEnd(char* str) {
return &str[i]; return &str[i];
} }
int64_t parseFraction(char* str, char** end, int32_t timePrec) { int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t i = 0; int32_t i = 0;
int64_t fraction = 0; int64_t fraction = 0;
@ -112,7 +114,7 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
int32_t totalLen = i; int32_t totalLen = i;
if (totalLen <= 0) { if (totalLen <= 0) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
/* parse the fraction */ /* parse the fraction */
@ -134,21 +136,24 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
} }
times = NANO_SEC_FRACTION_LEN - i; times = NANO_SEC_FRACTION_LEN - i;
} else { } else {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
fraction = strnatoi(str, i) * factor[times]; fraction = strnatoi(str, i) * factor[times];
*end = str + totalLen; *end = str + totalLen;
*pFraction = fraction;
return fraction; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t parseTimezone(char* str, int64_t* tzOffset) { int32_t parseTimezone(char* str, int64_t* tzOffset) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t hour = 0; int64_t hour = 0;
int32_t i = 0; int32_t i = 0;
if (str[i] != '+' && str[i] != '-') { if (str[i] != '+' && str[i] != '-') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
i++; i++;
@ -160,7 +165,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
continue; continue;
} }
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
char* sep = strchr(&str[i], ':'); char* sep = strchr(&str[i], ':');
@ -175,18 +180,18 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
} }
if (hour > 12 || hour < 0) { if (hour > 12 || hour < 0) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
// return error if there're illegal charaters after min(2 Digits) // return error if there're illegal charaters after min(2 Digits)
char* minStr = &str[i]; char* minStr = &str[i];
if (minStr[1] != '\0' && minStr[2] != '\0') { if (minStr[1] != '\0' && minStr[2] != '\0') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
int64_t minute = strnatoi(&str[i], 2); int64_t minute = strnatoi(&str[i], 2);
if (minute > 59 || (hour == 12 && minute > 0)) { if (minute > 59 || (hour == 12 && minute > 0)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
if (str[0] == '+') { if (str[0] == '+') {
@ -195,13 +200,13 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
*tzOffset = hour * 3600 + minute * 60; *tzOffset = hour * 3600 + minute * 60;
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) { if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) {
*offset = 0; *offset = 0;
return 0; return TSDB_CODE_SUCCESS;
} }
return parseTimezone(tzStr, offset); return parseTimezone(tzStr, offset);
} }
@ -219,6 +224,8 @@ int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
* 2013-04-12T15:52:01.123+0800 * 2013-04-12T15:52:01.123+0800
*/ */
int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t factor = TSDB_TICK_PER_SECOND(timePrec);
int64_t tzOffset = 0; int64_t tzOffset = 0;
@ -234,7 +241,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
} }
if (str == NULL) { if (str == NULL) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
/* mktime will be affected by TZ, set by using taos_options */ /* mktime will be affected by TZ, set by using taos_options */
@ -253,22 +260,18 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor; *time = seconds * factor;
} else if (str[0] == '.') { } else if (str[0] == '.') {
str += 1; str += 1;
if ((fraction = parseFraction(str, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str, &str, timePrec, &fraction));
return -1;
}
*time = seconds * factor + fraction; *time = seconds * factor + fraction;
char seg = str[0]; char seg = str[0];
if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') { if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') { } else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if (seg == '+' || seg == '-') { } else if (seg == '+' || seg == '-') {
// parse the timezone // parse the timezone
if (parseTimezone(str, &tzOffset) == -1) { TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
return -1;
}
*time += tzOffset * factor; *time += tzOffset * factor;
} }
@ -277,16 +280,14 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor + fraction; *time = seconds * factor + fraction;
// parse the timezone // parse the timezone
if (parseTimezone(str, &tzOffset) == -1) { TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
return -1;
}
*time += tzOffset * factor; *time += tzOffset * factor;
} else { } else {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static FORCE_INLINE bool validateTm(struct tm* pTm) { static FORCE_INLINE bool validateTm(struct tm* pTm) {
@ -314,6 +315,8 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) {
} }
int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0; *utime = 0;
struct tm tm = {0}; struct tm tm = {0};
@ -330,7 +333,7 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
// if parse failed, try "%Y-%m-%d" format // if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm); str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
@ -347,16 +350,16 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
if (*str == '.') { if (*str == '.') {
/* parse the second fraction part */ /* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
return -1;
}
} }
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) { int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0; *utime = 0;
struct tm tm = {0}; struct tm tm = {0};
tm.tm_isdst = -1; tm.tm_isdst = -1;
@ -374,7 +377,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
// if parse failed, try "%Y-%m-%d" format // if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm); str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
@ -384,13 +387,11 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
int64_t fraction = 0; int64_t fraction = 0;
if (*str == '.') { if (*str == '.') {
/* parse the second fraction part */ /* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) { TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
return -1;
}
} }
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction; *utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
char getPrecisionUnit(int32_t precision) { char getPrecisionUnit(int32_t precision) {
@ -482,10 +483,12 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre
// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double = // !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double =
// 1626006833631000064 // 1626006833631000064
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) { int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO && if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO &&
fromPrecision != TSDB_TIME_PRECISION_NANO) { fromPrecision != TSDB_TIME_PRECISION_NANO) {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
@ -541,15 +544,23 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
time *= factors[fromPrecision]; time *= factors[fromPrecision];
break; break;
default: { default: {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
} }
if (tmp >= (double)INT64_MAX) return INT64_MAX; if (tmp >= (double)INT64_MAX) {
if (tmp <= (double)INT64_MIN) return INT64_MIN; *pRes = INT64_MAX;
return time; } else if (tmp <= (double)INT64_MIN) {
*pRes = INT64_MIN;
} else {
*pRes = time;
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) { int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t charLen = varDataLen(inputData); int32_t charLen = varDataLen(inputData);
char* newColData; char* newColData;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
@ -558,7 +569,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return TSDB_CODE_INVALID_TIMESTAMP; TAOS_RETURN(TSDB_CODE_INVALID_TIMESTAMP);
} }
taosMemoryFree(newColData); taosMemoryFree(newColData);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
@ -566,50 +577,52 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData); int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData);
if (len < 0) { if (len < 0) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return TSDB_CODE_FAILED; TAOS_RETURN(TSDB_CODE_FAILED);
} }
newColData[len] = 0; newColData[len] = 0;
int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight); int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData); taosMemoryFree(newColData);
return ret; TAOS_RETURN(ret);
} }
taosMemoryFree(newColData); taosMemoryFree(newColData);
} else { } else {
return TSDB_CODE_FAILED; TAOS_RETURN(TSDB_CODE_FAILED);
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
switch (unit) { switch (unit) {
case 's': case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) { if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'm': case 'm':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'h': case 'h':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) { if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'd': case 'd':
if (val > INT64_MAX / MILLISECOND_PER_DAY) { if (val > INT64_MAX / MILLISECOND_PER_DAY) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
case 'w': case 'w':
if (val > INT64_MAX / MILLISECOND_PER_WEEK) { if (val > INT64_MAX / MILLISECOND_PER_WEEK) {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
(*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision); (*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision);
break; break;
@ -623,10 +636,10 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
(*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision); (*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision);
break; break;
default: { default: {
return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
} }
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
/* /*
@ -645,47 +658,50 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
*/ */
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit,
int32_t timePrecision) { int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0; errno = 0;
char* endPtr = NULL; char* endPtr = NULL;
/* get the basic numeric value */ /* get the basic numeric value */
int64_t timestamp = taosStr2Int64(token, &endPtr, 10); int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
if ((timestamp == 0 && token[0] != '0') || errno != 0) { if ((timestamp == 0 && token[0] != '0') || errno != 0) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
/* natual month/year are not allowed in absolute duration */ /* natual month/year are not allowed in absolute duration */
*unit = token[tokenlen - 1]; *unit = token[tokenlen - 1];
if (*unit == 'n' || *unit == 'y') { if (*unit == 'n' || *unit == 'y') {
return -1; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
return getDuration(timestamp, *unit, duration, timePrecision); return getDuration(timestamp, *unit, duration, timePrecision);
} }
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) { int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision,
bool negativeAllow) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0; errno = 0;
/* get the basic numeric value */ /* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10); *duration = taosStr2Int64(token, NULL, 10);
if ((*duration < 0 && !negativeAllow) || errno != 0) { if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
*unit = token[tokenLen - 1]; *unit = token[tokenLen - 1];
if (*unit == 'n' || *unit == 'y') { if (*unit == 'n' || *unit == 'y') {
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
if(isdigit(*unit)) { if (isdigit(*unit)) {
*unit = getPrecisionUnit(timePrecision); *unit = getPrecisionUnit(timePrecision);
} }
return getDuration(*duration, *unit, duration, timePrecision); return getDuration(*duration, *unit, duration, timePrecision);
} }
static bool taosIsLeapYear(int32_t year) { static bool taosIsLeapYear(int32_t year) { return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); }
return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0));
}
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
if (duration == 0) { if (duration == 0) {
@ -900,7 +916,8 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
// used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point // used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point
int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) { int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) {
if (pInterval->offset > 0) { if (pInterval->offset > 0) {
int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t slideStart =
taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision);
int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision); int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision);
return result; return result;
@ -960,7 +977,9 @@ const char* fmtts(int64_t ts) {
return buf; return buf;
} }
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) { int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
int32_t code = TSDB_CODE_SUCCESS;
char ts[40] = {0}; char ts[40] = {0};
struct tm ptm; struct tm ptm;
@ -996,17 +1015,18 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
default: default:
fractionLen = 0; fractionLen = 0;
return; TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} }
if (taosLocalTime(&quot, &ptm, buf) == NULL) { if (taosLocalTime(&quot, &ptm, buf) == NULL) {
return; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
length += snprintf(ts + length, fractionLen, format, mod); length += snprintf(ts + length, fractionLen, format, mod);
length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
tstrncpy(buf, ts, bufLen); tstrncpy(buf, ts, bufLen);
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) { int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) {
@ -1055,7 +1075,7 @@ typedef enum {
TSFKW_Mon, TSFKW_Mon,
TSFKW_MS, TSFKW_MS,
TSFKW_NS, TSFKW_NS,
//TSFKW_OF, // TSFKW_OF,
TSFKW_PM, TSFKW_PM,
TSFKW_P_M, TSFKW_P_M,
TSFKW_SS, TSFKW_SS,
@ -1076,7 +1096,7 @@ typedef enum {
TSFKW_day, TSFKW_day,
TSFKW_ddd, TSFKW_ddd,
TSFKW_dd, TSFKW_dd,
TSFKW_dy, // mon, tue TSFKW_dy, // mon, tue
TSFKW_d, TSFKW_d,
TSFKW_hh24, TSFKW_hh24,
TSFKW_hh12, TSFKW_hh12,
@ -1268,7 +1288,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
} }
if (*formatStr == '\\' && *(formatStr + 1)) { if (*formatStr == '\\' && *(formatStr + 1)) {
formatStr++; formatStr++;
last = NULL; // stop expanding last format, create new format last = NULL; // stop expanding last format, create new format
} }
if (last) { if (last) {
// expand // expand
@ -1289,7 +1309,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
// for other strings // for other strings
if (*formatStr == '\\' && *(formatStr + 1)) { if (*formatStr == '\\' && *(formatStr + 1)) {
formatStr++; formatStr++;
lastOtherFormat = NULL; // stop expanding lastOtherFormat = NULL; // stop expanding
} else { } else {
if (lastOtherFormat && !isSeperatorChar(*formatStr)) { if (lastOtherFormat && !isSeperatorChar(*formatStr)) {
// expanding // expanding
@ -1304,8 +1324,8 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
formatStr++; formatStr++;
} else { } else {
TSFormatNode format = { TSFormatNode format = {
.type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR, .type = isSeperatorChar(*formatStr) ? TS_FORMAT_NODE_TYPE_SEPARATOR : TS_FORMAT_NODE_TYPE_CHAR,
.key = NULL}; .key = NULL};
format.c = formatStr; format.c = formatStr;
format.len = 1; format.len = 1;
taosArrayPush(formats, &format); taosArrayPush(formats, &format);
@ -1318,7 +1338,7 @@ static void parseTsFormat(const char* formatStr, SArray* formats) {
} }
static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) { static int32_t tm2char(const SArray* formats, const struct STm* tm, char* s, int32_t outLen) {
int32_t size = taosArrayGetSize(formats); int32_t size = taosArrayGetSize(formats);
const char* start = s; const char* start = s;
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
TSFormatNode* format = taosArrayGet(formats, i); TSFormatNode* format = taosArrayGet(formats, i);
@ -1923,8 +1943,8 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr
return tm2char(*formats, &tm, out, outLen); return tm2char(*formats, &tm, out, outLen);
} }
int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg, int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision,
int32_t errMsgLen) { char* errMsg, int32_t errMsgLen) {
const char* sErrPos; const char* sErrPos;
int32_t fErrIdx; int32_t fErrIdx;
if (!*formats) { if (!*formats) {
@ -1977,26 +1997,27 @@ static int8_t UNIT_INDEX[26] = {/*a*/ 2, 0, -1, 6, -1, -1, -1,
#define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97] #define GET_UNIT_INDEX(idx) UNIT_INDEX[(idx) - 97]
static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h, d, w, month, y*/ static int64_t UNIT_MATRIX[10][11] = {/* ns, us, ms, s, min, h, d, w, month, y*/
/*ns*/ { 1, 1000, 0}, /*ns*/ {1, 1000, 0},
/*us*/ {1000, 1, 1000, 0}, /*us*/ {1000, 1, 1000, 0},
/*ms*/ { 0, 1000, 1, 1000, 0}, /*ms*/ {0, 1000, 1, 1000, 0},
/*s*/ { 0, 0, 1000, 1, 60, 0}, /*s*/ {0, 0, 1000, 1, 60, 0},
/*min*/ { 0, 0, 0, 60, 1, 60, 0}, /*min*/ {0, 0, 0, 60, 1, 60, 0},
/*h*/ { 0, 0, 0, 0, 60, 1, 1, 0}, /*h*/ {0, 0, 0, 0, 60, 1, 1, 0},
/*d*/ { 0, 0, 0, 0, 0, 24, 1, 7, 1, 0}, /*d*/ {0, 0, 0, 0, 0, 24, 1, 7, 1, 0},
/*w*/ { 0, 0, 0, 0, 0, 0, 7, 1, -1, 0}, /*w*/ {0, 0, 0, 0, 0, 0, 7, 1, -1, 0},
/*mon*/ { 0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0}, /*mon*/ {0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0},
/*y*/ { 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}}; /*y*/ {0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}};
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) { if (baseIdx == idx) {
if (interval < baseInterval) return false; if (interval < baseInterval) return false;
if (checkEq && interval == baseInterval) return false; if (checkEq && interval == baseInterval) return false;
return interval % baseInterval == 0; return interval % baseInterval == 0;
} }
int8_t next = baseIdx + 1; int8_t next = baseIdx + 1;
int64_t val = UNIT_MATRIX[baseIdx][next]; int64_t val = UNIT_MATRIX[baseIdx][next];
while (val != 0 && next <= idx) { while (val != 0 && next <= idx) {
if (val == -1) { if (val == -1) {
@ -2006,7 +2027,7 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
} }
if (val % baseInterval == 0 || baseInterval % val == 0) { if (val % baseInterval == 0 || baseInterval % val == 0) {
int8_t extra = baseInterval >= val ? 0 : 1; int8_t extra = baseInterval >= val ? 0 : 1;
bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1); bool needCheckEq = baseInterval >= val && !(baseIdx < next && val == 1);
if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) { if (!recursiveTsmaCheckRecursive(baseInterval / val + extra, next, interval, idx, needCheckEq && checkEq)) {
next++; next++;
val = UNIT_MATRIX[baseIdx][next]; val = UNIT_MATRIX[baseIdx][next];
@ -2021,7 +2042,8 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
return false; return false;
} }
static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) { static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false; if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) { if (baseIdx == idx) {
@ -2030,7 +2052,7 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
return interval % baseInterval == 0; return interval % baseInterval == 0;
} }
int8_t next = baseIdx - 1; int8_t next = baseIdx - 1;
int64_t val = UNIT_MATRIX[baseIdx][next]; int64_t val = UNIT_MATRIX[baseIdx][next];
while (val != 0 && next >= 0) { while (val != 0 && next >= 0) {
return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq); return recursiveTsmaCheckRecursiveReverse(baseInterval * val, next, interval, idx, checkEq);
@ -2041,18 +2063,27 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
/* /*
* @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit * @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
* @param baseInterval, baseUnit, interval/unit of base tsma * @param baseInterval, baseUnit, interval/unit of base tsma
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been * @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already
* translated to TICKS of [precision] * been translated to TICKS of [precision]
* @param unit the tsma unit going to create * @param unit the tsma unit going to create
* @param precision the precision of this db * @param precision the precision of this db
* @param checkEq pass true if same interval is not acceptable, false if acceptable. * @param checkEq pass true if same interval is not acceptable, false if acceptable.
* @ret true the tsma can be created, else cannot * @ret true the tsma can be created, else cannot
* */ * */
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) { bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision,
bool checkEq) {
bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit); bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit);
if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit); if (!baseIsCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit, &baseInterval)) {
return false;
}
}
bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit); bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit);
if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit); if (!isCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(interval, precision, unit, &interval)) {
return false;
}
}
bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq; bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq;

View File

@ -96,57 +96,58 @@ typedef struct STaskChkptInfo {
int8_t dropHTask; int8_t dropHTask;
}STaskChkptInfo; }STaskChkptInfo;
int32_t mndInitStream(SMnode *pMnode); int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode);
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName); int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream);
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream); int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId); int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId);
int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt); int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt);
bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock); bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock);
int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId); int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams); int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream); int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode, int32_t acceptCode); int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg); int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg, STrans **pTrans1);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo); void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans); int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream);
int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
int32_t mndProcessStreamHb(SRpcMsg *pReq); int32_t mndProcessStreamHb(SRpcMsg *pReq);
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
int32_t extractStreamNodeList(SMnode *pMnode); int32_t extractStreamNodeList(SMnode *pMnode);
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated); int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated);
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq); int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId, int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, int32_t taskId, int64_t checkpointId,
int64_t ts); int64_t ts);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter);
void destroyStreamTaskIter(SStreamTaskIter *pIter); void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); int32_t streamTaskIterGetCurrent(SStreamTaskIter *pIter, SStreamTask **pTask);
void mndInitExecInfo(); int32_t mndInitExecInfo();
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo); void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks); int32_t mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo);
void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo); void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo);
void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo); void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo);
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId); int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId);

View File

@ -31,17 +31,18 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow);
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq); static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) { int32_t mndInitShow(SMnode *pMnode) {
int32_t code = 0;
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show"); pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
if (pMgmt->cache == NULL) { if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr()); mError("failed to alloc show cache since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq); mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq);
return 0; TAOS_RETURN(code);
} }
void mndCleanupShow(SMnode *pMnode) { void mndCleanupShow(SMnode *pMnode) {
@ -212,6 +213,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
} }
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowObj *pShow = NULL; SShowObj *pShow = NULL;
@ -220,10 +222,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t rowsRead = 0; int32_t rowsRead = 0;
mDebug("mndProcessRetrieveSysTableReq start"); mDebug("mndProcessRetrieveSysTableReq start");
SRetrieveTableReq retrieveReq = {0}; SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb); mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
@ -232,17 +231,17 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (pMeta == NULL) { if (pMeta == NULL) {
pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) { if (pMeta == NULL) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
} }
pShow = mndCreateShowObj(pMnode, &retrieveReq); pShow = mndCreateShowObj(pMnode, &retrieveReq);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", terrstr()); mError("failed to process show-meta req since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
pShow->pMeta = pMeta; pShow->pMeta = pMeta;
@ -250,9 +249,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
if (pShow == NULL) { if (pShow == NULL) {
terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; code = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
} }
@ -264,9 +263,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
if (retrieveFp == NULL) { if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
@ -275,14 +274,16 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
} }
if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { code = -1;
return -1; if (retrieveReq.db[0] &&
(code = mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db)) != 0) {
TAOS_RETURN(code);
} }
if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) { if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) {
if(strcmp(pReq->info.conn.user, "root") != 0){ if(strcmp(pReq->info.conn.user, "root") != 0){
mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type); mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type);
terrno = TSDB_CODE_MND_NO_RIGHTS; code = TSDB_CODE_MND_NO_RIGHTS;
return -1; TAOS_RETURN(code);
} }
} }
@ -308,11 +309,11 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else { } else {
rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead); rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead);
if (rowsRead < 0) { if (rowsRead < 0) {
terrno = rowsRead; code = rowsRead;
mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true); mndReleaseShowObj(pShow, true);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; TAOS_RETURN(code);
} }
pBlock->info.rows = rowsRead; pBlock->info.rows = rowsRead;
@ -325,10 +326,10 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) { if (pRsp == NULL) {
mndReleaseShowObj(pShow, false); mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
return -1; TAOS_RETURN(code);
} }
pRsp->handle = htobe64(pShow->id); pRsp->handle = htobe64(pShow->id);

File diff suppressed because it is too large Load Diff

View File

@ -170,38 +170,54 @@ static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew
} }
static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0}; SDCreateSnodeReq createReq = {0};
createReq.dnodeId = pDnode->id; createReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq);
@ -212,23 +228,24 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_CREATE_SNODE; action.msgType = TDMT_DND_CREATE_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +256,12 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_SNODE; action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) { if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) {
@ -256,23 +273,27 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
snodeObj.updateTime = snodeObj.createdTime; snodeObj.updateTime = snodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeRedoLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeUndoLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeCommitLogs(pTrans, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
if (mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
@ -282,32 +303,27 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL; SDnodeObj *pDnode = NULL;
SMCreateSnodeReq createReq = {0}; SMCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("snode:%d, start to create", createReq.dnodeId); mInfo("snode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE), NULL, _OVER);
goto _OVER;
}
// pObj = mndAcquireSnode(pMnode, createReq.dnodeId); // pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
// if (pObj != NULL) { // if (pObj != NULL) {
// terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; // terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
// goto _OVER; // goto _OVER;
// } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { // } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) {
// goto _OVER; // goto _OVER;
// } // }
if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){ if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){
terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; code = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER; goto _OVER;
} }
@ -316,41 +332,52 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); mError("snode:%d, failed to create since %s", createReq.dnodeId, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
// mndReleaseSnode(pMnode, pObj); // mndReleaseSnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq); tFreeSMCreateQnodeReq(&createReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0}; SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id; dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen); void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -361,20 +388,20 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
action.msgType = TDMT_DND_DROP_SNODE; action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) { int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) {
if (pObj == NULL) return 0; if (pObj == NULL) return 0;
if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeRedoLogs(pTrans, pObj));
if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeCommitLogs(pTrans, pObj));
if (!force) { if (!force) {
if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; TAOS_CHECK_RETURN(mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj));
} }
return 0; return 0;
} }
@ -383,18 +410,22 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans); mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER; TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
@ -403,23 +434,20 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
SSnodeObj *pObj = NULL; SSnodeObj *pObj = NULL;
SMDropSnodeReq dropReq = {0}; SMDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("snode:%d, start to drop", dropReq.dnodeId); mInfo("snode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE), NULL, _OVER);
goto _OVER;
}
if (dropReq.dnodeId <= 0) { if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _OVER; goto _OVER;
} }
pObj = mndAcquireSnode(pMnode, dropReq.dnodeId); pObj = mndAcquireSnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) { if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -429,12 +457,12 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); mError("snode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
} }
mndReleaseSnode(pMnode, pObj); mndReleaseSnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq); tFreeSMCreateQnodeReq(&dropReq);
return code; TAOS_RETURN(code);
} }
static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

File diff suppressed because it is too large Load Diff

View File

@ -134,17 +134,18 @@ int32_t mndInitStream(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
mndInitExecInfo(); int32_t code = mndInitExecInfo();
if (code) {
if (sdbSetTable(pMnode->pSdb, table) != 0) { return code;
return -1;
} }
if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { code = sdbSetTable(pMnode->pSdb, table);
return -1; if (code) {
return terrno;
} }
return 0; code = sdbSetTable(pMnode->pSdb, tableSeq);
return code;
} }
void mndCleanupStream(SMnode *pMnode) { void mndCleanupStream(SMnode *pMnode) {
@ -251,13 +252,15 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
return 0; return 0;
} }
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream) {
SSdb *pSdb = pMnode->pSdb; terrno = 0;
SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { SSdb *pSdb = pMnode->pSdb;
(*pStream) = sdbAcquire(pSdb, SDB_STREAM, streamName);
if ((*pStream) == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
} }
return pStream; return terrno;
} }
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
@ -530,9 +533,21 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
} }
int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) { int32_t mndPersistStreamTasks(STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create task iter for stream:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
}
if (mndPersistTaskDeployReq(pTrans, pTask) < 0) { if (mndPersistTaskDeployReq(pTrans, pTask) < 0) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return -1;
@ -706,7 +721,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
char *sql = NULL; char *sql = NULL;
int32_t sqlLen = 0; int32_t sqlLen = 0;
const char *pMsg = "create stream tasks on dnodes"; const char *pMsg = "create stream tasks on dnodes";
int32_t code = 0;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
SCMCreateStreamReq createReq = {0}; SCMCreateStreamReq createReq = {0};
@ -726,8 +741,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
pStream = mndAcquireStream(pMnode, createReq.name); code = mndAcquireStream(pMnode, createReq.name, &pStream);
if (pStream != NULL) { if (pStream != NULL || code == 0) {
if (createReq.igExists) { if (createReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createReq.name); mInfo("stream:%s, already exist, ignore exist is set", createReq.name);
goto _OVER; goto _OVER;
@ -760,8 +775,9 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg); STrans *pTrans = NULL;
if (pTrans == NULL) { code = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg, &pTrans);
if (pTrans == NULL || code) {
goto _OVER; goto _OVER;
} }
@ -802,11 +818,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
// add into buffer firstly // add into buffer firstly
// to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name); mDebug("stream stream:%s start to register tasks into task nodeList and set initial checkpointId", createReq.name);
saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo);
// mndRegisterConsensusChkptId(execInfo.pStreamConsensus, streamObj.uid); streamMutexUnlock(&execInfo.lock);
taosThreadMutexUnlock(&execInfo.lock);
// execute creation // execute creation
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
@ -867,7 +882,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
{ // check the max checkpoint id from all vnodes. { // check the max checkpoint id from all vnodes.
int64_t maxCheckpointId = -1; int64_t maxCheckpointId = -1;
if (lock) { if (lock) {
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
} }
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
@ -888,7 +903,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
} }
if (lock) { if (lock) {
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
} }
if (maxCheckpointId > maxChkptId) { if (maxCheckpointId > maxChkptId) {
@ -989,11 +1004,13 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, STrans *pTrans = NULL;
"gen checkpoint for stream"); code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME,
if (pTrans == NULL) { "gen checkpoint for stream", &pTrans);
if (pTrans == NULL || code) {
code = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); tstrerror(code));
goto _ERR; goto _ERR;
} }
@ -1033,7 +1050,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) { if ((code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY)) != TSDB_CODE_SUCCESS) {
return code; goto _ERR;
} }
if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) { if ((code = mndTransPrepare(pMnode, pTrans)) != TSDB_CODE_SUCCESS) {
@ -1057,13 +1074,13 @@ int32_t extractStreamNodeList(SMnode *pMnode) {
static bool taskNodeIsUpdated(SMnode *pMnode) { static bool taskNodeIsUpdated(SMnode *pMnode) {
// check if the node update happens or not // check if the node update happens or not
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int32_t numOfNodes = extractStreamNodeList(pMnode); int32_t numOfNodes = extractStreamNodeList(pMnode);
if (numOfNodes == 0) { if (numOfNodes == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
execInfo.ts = taosGetTimestampSec(); execInfo.ts = taosGetTimestampSec();
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return false; return false;
} }
@ -1071,17 +1088,22 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i);
if (pNodeEntry->stageUpdated) { if (pNodeEntry->stageUpdated) {
mDebug("stream task not ready due to node update detected, checkpoint not issued"); mDebug("stream task not ready due to node update detected, checkpoint not issued");
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return true; return true;
} }
} }
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
mWarn("not all vnodes ready, quit from vnodes status check"); mWarn("not all vnodes ready, quit from vnodes status check");
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return true; return true;
} }
@ -1097,7 +1119,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
mDebug("stream tasks not ready due to node update"); mDebug("stream tasks not ready due to node update");
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return nodeUpdated; return nodeUpdated;
} }
@ -1107,7 +1129,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
return -1; return -1;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
if (taosArrayGetSize(execInfo.pNodeList) == 0) { if (taosArrayGetSize(execInfo.pNodeList) == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
@ -1152,7 +1174,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
removeTasksInBuf(pInvalidList, &execInfo); removeTasksInBuf(pInvalidList, &execInfo);
taosArrayDestroy(pInvalidList); taosArrayDestroy(pInvalidList);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return ready ? 0 : -1; return ready ? 0 : -1;
} }
@ -1215,14 +1237,14 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
continue; continue;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid); int64_t startTs = getStreamTaskLastReadyState(execInfo.pTaskList, pStream->uid);
if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) { if (startTs != -1 && (now - startTs) < tsStreamCheckpointInterval * 1000) {
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
continue; continue;
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration}; SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration};
taosArrayPush(pList, &in); taosArrayPush(pList, &in);
@ -1265,8 +1287,9 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
for (int32_t i = 0; i < numOfQual; ++i) { for (int32_t i = 0; i < numOfQual; ++i) {
SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i); SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i);
SStreamObj *p = mndGetStreamObj(pMnode, pCheckpointInfo->streamId); SStreamObj *p = NULL;
if (p != NULL) { code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p);
if (p != NULL && code == 0) {
code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true);
sdbRelease(pSdb, p); sdbRelease(pSdb, p);
@ -1289,6 +1312,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
SMDropStreamReq dropReq = {0}; SMDropStreamReq dropReq = {0};
if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) {
@ -1299,8 +1323,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
mDebug("recv drop stream:%s msg", dropReq.name); mDebug("recv drop stream:%s msg", dropReq.name);
pStream = mndAcquireStream(pMnode, dropReq.name); code = mndAcquireStream(pMnode, dropReq.name, &pStream);
if (pStream == NULL) { if (pStream == NULL || code != 0) {
if (dropReq.igNotExists) { if (dropReq.igNotExists) {
mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name); mInfo("stream:%s not exist, ignore not exist is set, drop stream exec done with success", dropReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1356,15 +1380,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = NULL;
if (pTrans == NULL) { code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code) {
mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr()); mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq); tFreeMDropStreamReq(&dropReq);
return -1; return -1;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pStream->uid);
// drop all tasks // drop all tasks
if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) { if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
@ -1857,9 +1882,9 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfRows = 0; int32_t numOfRows = 0;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo); mndInitStreamExecInfo(pMnode, &execInfo);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
while (numOfRows < rowsCapacity) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
@ -1876,11 +1901,24 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
} }
// add row for each task // add row for each task
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
while (streamTaskIterNextTask(pIter)) { int32_t code = createStreamTaskIter(pStream, &pIter);
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); if (code) {
taosRUnLockLatch(&pStream->lock);
sdbRelease(pSdb, pStream);
mError("failed to create task iter for stream:%s", pStream->name);
continue;
}
int32_t code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows); while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
break;
}
code = setTaskAttrInResBlock(pStream, pTask, pBlock, numOfRows);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
numOfRows++; numOfRows++;
} }
@ -1906,6 +1944,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter) {
static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
SMPauseStreamReq pauseReq = {0}; SMPauseStreamReq pauseReq = {0};
if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) { if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) {
@ -1913,9 +1952,8 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
pStream = mndAcquireStream(pMnode, pauseReq.name); code = mndAcquireStream(pMnode, pauseReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (pStream == NULL) {
if (pauseReq.igNotExists) { if (pauseReq.igNotExists) {
mInfo("stream:%s, not exist, not pause stream", pauseReq.name); mInfo("stream:%s, not exist, not pause stream", pauseReq.name);
return 0; return 0;
@ -1955,7 +1993,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
{ // check for tasks, if tasks are not ready, not allowed to pause { // check for tasks, if tasks are not ready, not allowed to pause
bool found = false; bool found = false;
bool readyToPause = true; bool readyToPause = true;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
@ -1978,7 +2016,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
found = true; found = true;
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
if (!found) { if (!found) {
mError("stream:%s task not report status yet, not ready for pause", pauseReq.name); mError("stream:%s task not report status yet, not ready for pause", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1992,42 +2030,49 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
} }
} }
STrans *pTrans = STrans *pTrans = NULL;
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream"); code = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream", &pTrans);
if (pTrans == NULL) { if (pTrans == NULL || code) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_PAUSE_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
// if nodeUpdate happened, not send pause trans // if nodeUpdate happened, not send pause trans
if (mndStreamSetPauseAction(pMnode, pTrans, pStream) < 0) { code = mndStreamSetPauseAction(pMnode, pTrans, pStream);
if (code) {
mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr()); mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// pause stream // pause stream
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
pStream->status = STREAM_STATUS__PAUSE; pStream->status = STREAM_STATUS__PAUSE;
if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY) < 0) { code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code) {
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2039,6 +2084,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return -1; return -1;
@ -2050,9 +2096,8 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
pStream = mndAcquireStream(pMnode, resumeReq.name); code = mndAcquireStream(pMnode, resumeReq.name, &pStream);
if (pStream == NULL || code != 0) {
if (pStream == NULL) {
if (resumeReq.igNotExists) { if (resumeReq.igNotExists) {
mInfo("stream:%s not exist, not resume stream", resumeReq.name); mInfo("stream:%s not exist, not resume stream", resumeReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2081,22 +2126,28 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = STrans *pTrans = NULL;
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream"); code =
if (pTrans == NULL) { doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream", &pTrans);
if (pTrans == NULL || code) {
mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_RESUME_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
// set the resume action // set the resume action
if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) { if (mndStreamSetResumeAction(pTrans, pMnode, pStream, resumeReq.igUntreated) < 0) {
mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to drop task since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// resume stream // resume stream
@ -2107,7 +2158,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
@ -2115,7 +2166,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare pause stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -2189,6 +2240,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
STrans *pTrans = NULL; STrans *pTrans = NULL;
int32_t code = 0;
// conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool // conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool
while (1) { while (1) {
@ -2215,12 +2267,11 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
// here create only one trans // here create only one trans
if (pTrans == NULL) { if (pTrans == NULL) {
pTrans = code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets", &pTrans);
doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets"); if (pTrans == NULL || code) {
if (pTrans == NULL) {
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return terrno; return terrno = code;
} }
mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid); mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_UPDATE_NAME, pStream->uid);
@ -2237,7 +2288,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid, mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
pStream->name, pTrans->id); pStream->name, pTrans->id);
int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans); code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans);
// todo: not continue, drop all and retry again // todo: not continue, drop all and retry again
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -2252,7 +2303,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; return code;
} }
} }
@ -2261,16 +2312,17 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0; return 0;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
@ -2287,9 +2339,21 @@ static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) {
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
SStreamTaskIter *pTaskIter = createStreamTaskIter(pStream); SStreamTaskIter *pTaskIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pTaskIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
sdbRelease(pSdb, pStream);
mError("failed to create task iter for stream:%s", pStream->name);
continue;
}
while (streamTaskIterNextTask(pTaskIter)) { while (streamTaskIterNextTask(pTaskIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pTaskIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pTaskIter, &pTask);
if (code) {
break;
}
SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; SNodeEntry entry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId};
epsetAssign(&entry.epset, &pTask->info.epSet); epsetAssign(&entry.epset, &pTask->info.epSet);
@ -2336,9 +2400,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
int32_t numOfNodes = extractStreamNodeList(pMnode); int32_t numOfNodes = extractStreamNodeList(pMnode);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
if (numOfNodes == 0) { if (numOfNodes == 0) {
mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing"); mDebug("end to do stream task(s) node change checking, no stream tasks exist, do nothing");
@ -2348,7 +2412,13 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
} }
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
if (code) {
mError("failed to take the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
atomic_store_32(&mndNodeCheckSentinel, 0); atomic_store_32(&mndNodeCheckSentinel, 0);
@ -2356,7 +2426,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
return 0; return 0;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot);
@ -2380,7 +2450,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
} }
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(changeInfo.pUpdateNodeList); taosArrayDestroy(changeInfo.pUpdateNodeList);
taosHashCleanup(changeInfo.pDBMap); taosHashCleanup(changeInfo.pDBMap);
@ -2406,9 +2476,19 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
} }
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create task iter for stream:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
break;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
@ -2478,10 +2558,11 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId); mDebug("receive stream task checkpoint req msg, vgId:%d, s-task:0x%x", req.nodeId, req.taskId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream);
if (pStream == NULL || code != 0) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf", mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf",
req.streamId); req.streamId);
@ -2492,7 +2573,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return -1; return -1;
} else { } else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
@ -2537,7 +2618,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
{ {
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)}; SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)};
@ -2597,10 +2678,11 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId); req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { int32_t code = mndGetStreamObj(pMnode, req.streamId, &pStream);
if (pStream == NULL || code != 0) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId); mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer // not in meta-store yet, try to acquire the task in exec buffer
@ -2610,7 +2692,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
return -1; return -1;
} else { } else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
@ -2642,7 +2724,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS); doSendQuickRsp(&pReq->info, sizeof(SMStreamUpdateChkptRsp), req.nodeId, TSDB_CODE_SUCCESS);
return 0; return 0;
@ -2707,7 +2789,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// req.nodeId, req.streamId, req.taskId, req.checkpointId); // req.nodeId, req.streamId, req.taskId, req.checkpointId);
// //
// // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. // // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
// taosThreadMutexLock(&execInfo.lock); // streamMutexLock(&execInfo.lock);
// //
// // mnode handle the create stream transaction too slow may cause this problem // // mnode handle the create stream transaction too slow may cause this problem
// SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); // SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
@ -2721,7 +2803,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// if (p == NULL) { // if (p == NULL) {
// mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId); // mError("failed to find the stream:0x%" PRIx64 " in buf, not handle consensus-checkpointId", req.streamId);
// terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; // terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// //
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return -1; // return -1;
@ -2737,7 +2819,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// //
// int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream); // int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
// if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly // if ((pStream != NULL) && (pStream->checkpointId == 0)) { // not generated checkpoint yet, return 0 directly
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, 0, req.startTs);
// //
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
@ -2754,7 +2836,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks); // SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, req.streamId, numOfTasks);
// mndAddConsensusTasks(pInfo, &req); // mndAddConsensusTasks(pInfo, &req);
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
// } // }
@ -2764,7 +2846,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId); // req.nodeId, req.streamId, pStream->name, chkId, pStream->checkpointId);
// mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs); // mndCreateSetConsensusChkptIdTrans(pMnode, pStream, req.taskId, chkId, req.startTs);
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
// } // }
@ -2777,7 +2859,7 @@ static void doSendQuickRsp(SRpcHandleInfo *pInfo, int32_t msgSize, int32_t vgId,
// mndReleaseStream(pMnode, pStream); // mndReleaseStream(pMnode, pStream);
// } // }
// //
// taosThreadMutexUnlock(&execInfo.lock); // streamMutexUnlock(&execInfo.lock);
// doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno); // doSendQuickRsp(&pMsg->info, sizeof(SMStreamReqConsensChkptRsp), req.nodeId, terrno);
// return 0; // return 0;
//} //}
@ -2790,15 +2872,21 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
mDebug("start to process consensus-checkpointId in tmr"); mDebug("start to process consensus-checkpointId in tmr");
bool allReady = true; bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *pNodeSnapshot = NULL;
int32_t code = mndTakeVgroupSnapshot(pMnode, &allReady, &pNodeSnapshot);
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
if (!allReady) { if (!allReady) {
mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process"); mWarn("not all vnodes are ready, end to process the consensus-checkpointId in tmr process");
taosArrayDestroy(pStreamList); taosArrayDestroy(pStreamList);
return 0; return 0;
} }
taosThreadMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
void *pIter = NULL; void *pIter = NULL;
while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.pStreamConsensus, pIter)) != NULL) {
@ -2808,8 +2896,9 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
int32_t num = taosArrayGetSize(pInfo->pTaskList); int32_t num = taosArrayGetSize(pInfo->pTaskList);
SArray *pList = taosArrayInit(4, sizeof(int32_t)); SArray *pList = taosArrayInit(4, sizeof(int32_t));
SStreamObj *pStream = mndGetStreamObj(pMnode, pInfo->streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { // stream has been dropped already code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) { // stream has been dropped already
mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId); mDebug("stream:0x%" PRIx64 " dropped already, continue", pInfo->streamId);
taosArrayDestroy(pList); taosArrayDestroy(pList);
continue; continue;
@ -2868,14 +2957,14 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) {
int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i); int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i);
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId); code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId);
} }
taosThreadMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
taosArrayDestroy(pStreamList); taosArrayDestroy(pStreamList);
mDebug("end to process consensus-checkpointId in tmr"); mDebug("end to process consensus-checkpointId in tmr");
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) { static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
@ -2926,32 +3015,41 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
} }
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) { int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, STrans *pTrans = NULL;
"update checkpoint-info"); int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME,
if (pTrans == NULL) { "update checkpoint-info", &pTrans);
return terrno; if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
} }
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); if (code){
if (code != 0) { sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
} }
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) { if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);

View File

@ -61,15 +61,23 @@ void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
} }
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, STrans *pTrans = NULL;
" reset from failed checkpoint"); int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
if (pTrans == NULL) { " reset from failed checkpoint", &pTrans);
if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return terrno; return terrno;
} }
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); if (code) {
if (code != 0) { sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
@ -79,14 +87,15 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -99,8 +108,9 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t t
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
mndKillTransImpl(pMnode, transId, ""); mndKillTransImpl(pMnode, transId, "");
SStreamObj *pStream = mndGetStreamObj(pMnode, streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { code = mndGetStreamObj(pMnode, streamId, &pStream);
if (pStream == NULL || code != 0) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST; code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid);
} else { } else {
@ -159,34 +169,39 @@ int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) {
} }
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); STrans *pTrans = NULL;
if (pTrans == NULL) { int32_t code =
doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code != 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1; return code;
} }
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
if (code) {
return code;
}
// drop all tasks // drop all tasks
if (mndStreamSetDropActionFromList(pMnode, pTrans, pList) < 0) { if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, pList)) < 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
// drop stream // drop stream
if (mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED) < 0) { if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) {
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return code;
} }
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
@ -228,10 +243,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SStreamHbMsg req = {0}; SStreamHbMsg req = {0};
SArray *pFailedChkpt = NULL; SArray *pFailedChkpt = NULL;
SArray *pOrphanTasks = NULL; SArray *pOrphanTasks = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if (suspendAllStreams(pMnode, &pReq->info) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) {
return -1; return code;
} }
} }
@ -241,8 +257,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (tDecodeStreamHbMsg(&decoder, &req) < 0) { if (tDecodeStreamHbMsg(&decoder, &req) < 0) {
tCleanupStreamHbMsg(&req); tCleanupStreamHbMsg(&req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG; code = terrno = TSDB_CODE_INVALID_MSG;
return -1; return code;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -257,12 +273,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (!validateHbMsg(execInfo.pNodeList, req.vgId)) { if (!validateHbMsg(execInfo.pNodeList, req.vgId)) {
mError("vgId:%d not exists in nodeList buf, discarded", req.vgId); mError("vgId:%d not exists in nodeList buf, discarded", req.vgId);
terrno = TSDB_CODE_INVALID_MSG; code = terrno = TSDB_CODE_INVALID_MSG;
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return -1; return code;
} }
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
@ -293,11 +309,23 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
.startTs = pChkInfo->consensusTs, .startTs = pChkInfo->consensusTs,
}; };
SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId); SStreamObj *pStream = NULL;
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream); code = mndGetStreamObj(pMnode, p->id.streamId, &pStream);
if (code) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
continue;
}
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
SCheckpointConsensusInfo *pInfo = NULL;
code = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks, &pInfo);
if (code == 0) {
mndAddConsensusTasks(pInfo, &cp);
} else {
mError("failed to get consensus checkpoint-info");
}
SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks);
mndAddConsensusTasks(pInfo, &cp);
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
} }
@ -338,9 +366,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
// kill the checkpoint trans and then set all tasks status to be normal // kill the checkpoint trans and then set all tasks status to be normal
if (taosArrayGetSize(pFailedChkpt) > 0) { if (taosArrayGetSize(pFailedChkpt) > 0) {
bool allReady = true; bool allReady = true;
if (pMnode != NULL) { if (pMnode != NULL) {
SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); SArray *p = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &p);
taosArrayDestroy(p); taosArrayDestroy(p);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
} else { } else {
allReady = false; allReady = false;
} }
@ -374,7 +408,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId); doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks); cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return TSDB_CODE_SUCCESS; return terrno;
} }
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg

View File

@ -153,27 +153,30 @@ int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) {
return 0; return 0;
} }
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg) { const char *pMsg, STrans ** pTrans1) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); *pTrans1 = NULL;
if (pTrans == NULL) { terrno = 0;
STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (p == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return terrno;
} }
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName); mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, p) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT; terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno)); mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
mndTransDrop(pTrans); mndTransDrop(p);
return NULL; return terrno;
} }
terrno = 0; *pTrans1 = p;
return pTrans; return 0;
} }
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@ -272,8 +275,9 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) {
continue; continue;
} }
SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId); SStreamObj *pStream = NULL;
if (pStream != NULL) { int32_t code = mndGetStreamObj(pMnode, pTransInfo->streamId, &pStream);
if (pStream != NULL && code == 0) {
if (identicalName(pStream->sourceDb, pDBName, len)) { if (identicalName(pStream->sourceDb, pDBName, len)) {
mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb); mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb);
} else if (identicalName(pStream->targetDb, pDBName, len)) { } else if (identicalName(pStream->targetDb, pDBName, len)) {

View File

@ -28,20 +28,20 @@ struct SStreamTaskIter {
int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId); int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId);
SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) { int32_t createStreamTaskIter(SStreamObj* pStream, SStreamTaskIter** pIter) {
SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter)); *pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
if (pIter == NULL) { if (*pIter == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return terrno;
} }
pIter->level = -1; (*pIter)->level = -1;
pIter->ordinalIndex = 0; (*pIter)->ordinalIndex = 0;
pIter->pStream = pStream; (*pIter)->pStream = pStream;
pIter->totalLevel = taosArrayGetSize(pStream->tasks); (*pIter)->totalLevel = taosArrayGetSize(pStream->tasks);
pIter->pTask = NULL; (*pIter)->pTask = NULL;
return pIter; return 0;
} }
bool streamTaskIterNextTask(SStreamTaskIter* pIter) { bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
@ -72,19 +72,27 @@ bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
return false; return false;
} }
SStreamTask* streamTaskIterGetCurrent(SStreamTaskIter* pIter) { int32_t streamTaskIterGetCurrent(SStreamTaskIter* pIter, SStreamTask** pTask) {
return pIter->pTask; if (pTask) {
*pTask = pIter->pTask;
if (*pTask != NULL) {
return TSDB_CODE_SUCCESS;
}
}
return TSDB_CODE_INVALID_PARA;
} }
void destroyStreamTaskIter(SStreamTaskIter* pIter) { void destroyStreamTaskIter(SStreamTaskIter* pIter) {
taosMemoryFree(pIter); taosMemoryFree(pIter);
} }
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
int32_t replica = -1; // do the replica check int32_t replica = -1; // do the replica check
int32_t code = 0;
*allReady = true; *allReady = true;
SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry)); SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry));
@ -131,10 +139,15 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
} }
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf)); (void) epsetToStr(&entry.epset, buf, tListLen(buf));
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
}
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -145,43 +158,57 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
break; break;
} }
SNodeEntry entry = {0}; SNodeEntry entry = {.nodeId = SNODE_HANDLE};
addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port);
entry.nodeId = SNODE_HANDLE; if (code) {
sdbRelease(pSdb, pObj);
continue;
}
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf)); (void) epsetToStr(&entry.epset, buf, tListLen(buf));
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
}
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pObj); sdbRelease(pSdb, pObj);
} }
return pVgroupList; *pList = pVgroupList;
return code;
} }
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) { int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj **pStream) {
void *pIter = NULL; void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; *pStream = NULL;
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { SStreamObj *p = NULL;
if (pStream->uid == streamId) { while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&p)) != NULL) {
if (p->uid == streamId) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return pStream; *pStream = p;
return TSDB_CODE_SUCCESS;
} }
sdbRelease(pSdb, pStream); sdbRelease(pSdb, p);
} }
return NULL; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) { void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) {
STrans *pTrans = mndAcquireTrans(pMnode, transId); STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans != NULL) { if (pTrans != NULL) {
mInfo("kill active transId:%d in Db:%s", transId, pDbName); mInfo("kill active transId:%d in Db:%s", transId, pDbName);
mndKillTrans(pMnode, pTrans); int32_t code = mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans);
if (code) {
mError("failed to kill trans:%d", pTrans->id);
}
} else { } else {
mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId); mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId);
} }
@ -197,11 +224,16 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj);
if (pIter != NULL) { if (pIter != NULL) {
addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); int32_t code = addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
sdbRelease(pMnode->pSdb, pObj); sdbRelease(pMnode->pSdb, pObj);
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
*hasEpset = true; if (code) {
return TSDB_CODE_SUCCESS; *hasEpset = false;
mError("failed to set epset");
} else {
*hasEpset = true;
}
return code;
} else { } else {
mError("failed to acquire snode epset"); mError("failed to acquire snode epset");
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
@ -223,12 +255,14 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
} }
static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) {
terrno = 0;
SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -242,31 +276,45 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { if (code != TSDB_CODE_SUCCESS || (!hasEpset)) {
terrno = code; terrno = code;
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return terrno;
} }
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return terrno;
} }
mDebug("set the resume action for trans:%d", pTrans->id); mDebug("set the resume action for trans:%d", pTrans->id);
return 0; return 0;
} }
SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); *pTask = NULL;
SStreamTask *p = NULL;
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); code = streamTaskIterGetCurrent(pIter, &p);
if (pTask->id.taskId == pId->taskId) { if (code) {
continue;
}
if (p->id.taskId == pId->taskId) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return pTask; *pTask = p;
return 0;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return NULL; return TSDB_CODE_FAILED;
} }
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) { int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
@ -280,13 +328,25 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
} }
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) { int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code || pTask == NULL) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetResumeAction(pTrans, pMnode, pTask, igUntreated);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) { if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) {
@ -303,7 +363,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq), mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -320,25 +380,38 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
} }
char buf[256] = {0}; char buf[256] = {0};
epsetToStr(&epset, buf, tListLen(buf)); (void) epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
} }
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetPauseAction(pMnode, pTrans, pTask) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetPauseAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) { if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) {
@ -348,14 +421,14 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return 0; return code;
} }
static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->info.nodeId); pReq->head.vgId = htonl(pTask->info.nodeId);
@ -366,28 +439,40 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction
terrno = code; return code;
return -1;
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
} }
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream); SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while(streamTaskIterNextTask(pIter)) { while(streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
if (doSetDropAction(pMnode, pTrans, pTask) < 0) { code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
return -1; return code;
}
code = doSetDropAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
@ -398,7 +483,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) { if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return terrno;
} }
pReq->head.vgId = htonl(pTask->nodeId); pReq->head.vgId = htonl(pTask->nodeId);
@ -409,16 +494,15 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId); int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId);
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction
terrno = code;
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return code;
} }
return 0; return 0;
@ -427,19 +511,35 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) { int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) {
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SOrphanTask* pTask = taosArrayGet(pList, i); SOrphanTask* pTask = taosArrayGet(pList, i);
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId); int32_t code = doSetDropActionFromId(pMnode, pTrans, pTask);
doSetDropActionFromId(pMnode, pTrans, pTask); if (code != 0) {
return code;
} else {
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId);
}
} }
return 0; return 0;
} }
static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId, static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
int32_t transId) { int32_t transId) {
int32_t code = 0;
pMsg->streamId = pId->streamId; pMsg->streamId = pId->streamId;
pMsg->taskId = pId->taskId; pMsg->taskId = pId->taskId;
pMsg->transId = transId; pMsg->transId = transId;
pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo)); pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList); if (pMsg->pNodeList == NULL) {
mError("failed to prepare node list, code:out of memory");
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code == 0) {
void *p = taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
if (p == NULL) {
mError("failed to add update node list into nodeList");
}
}
} }
static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
@ -454,7 +554,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return -1; return terrno;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
@ -463,13 +563,18 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return -1; return terrno;
} }
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
tEncodeStreamTaskUpdateMsg(&encoder, &req); code = tEncodeStreamTaskUpdateMsg(&encoder, &req);
if (code == -1) {
tEncoderClear(&encoder);
taosArrayDestroy(req.pNodeList);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)buf; SMsgHead *pMsgHead = (SMsgHead *)buf;
pMsgHead->contLen = htonl(tlen); pMsgHead->contLen = htonl(tlen);
@ -487,15 +592,20 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) { static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
void *pBuf = NULL; void *pBuf = NULL;
int32_t len = 0; int32_t len = 0;
streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); int32_t code = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
if (code) {
return code;
}
doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
if (code) {
return code;
}
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { if (code != TSDB_CODE_SUCCESS || !hasEpset) {
terrno = code;
return code; return code;
} }
@ -510,16 +620,30 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
// build trans to update the epset // build trans to update the epset
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) { int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid); mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo); code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
@ -558,16 +682,30 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
} }
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock); SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
int32_t code = doSetResetAction(pMnode, pTrans, pTask); code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetResetAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
@ -581,8 +719,12 @@ static void freeTaskList(void* param) {
taosArrayDestroy(*pList); taosArrayDestroy(*pList);
} }
void mndInitExecInfo() { int32_t mndInitExecInfo() {
taosThreadMutexInit(&execInfo.lock, NULL); int32_t code = taosThreadMutexInit(&execInfo.lock, NULL);
if (code) {
return code;
}
_hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR); _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
@ -596,6 +738,7 @@ void mndInitExecInfo() {
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList); taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList); taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList);
return 0;
} }
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
@ -608,7 +751,10 @@ void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
for (int32_t j = 0; j < size; ++j) { for (int32_t j = 0; j < size; ++j) {
SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
if (pEntry->nodeId == p->nodeId) { if (pEntry->nodeId == p->nodeId) {
taosArrayPush(pValidList, p); void* px = taosArrayPush(pValidList, p);
if (px == NULL) {
mError("failed to put node into list, nodeId:%d", p->nodeId);
}
break; break;
} }
} }
@ -626,7 +772,10 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); int32_t code = taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
if (code) {
return code;
}
for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
@ -645,28 +794,45 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) { void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) {
STaskId *pId = taosArrayGet(pTaskIds, i); STaskId *pId = taosArrayGet(pTaskIds, i);
doRemoveTasks(pExecInfo, pId); int32_t code = doRemoveTasks(pExecInfo, pId);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, pId->taskId);
}
} }
} }
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexLock(&pExecNode->lock); SStreamTaskIter *pIter = NULL;
streamMutexLock(&pExecNode->lock);
// 1. remove task entries // 1. remove task entries
SStreamTaskIter *pIter = createStreamTaskIter(pStream); int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
streamMutexUnlock(&pExecNode->lock);
mError("failed to create stream task iter:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
continue;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
doRemoveTasks(pExecNode, &id); code = doRemoveTasks(pExecNode, &id);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, id.taskId);
}
} }
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
// 2. remove stream entry in consensus hash table // 2. remove stream entry in consensus hash table
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid); (void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid);
taosThreadMutexUnlock(&pExecNode->lock); streamMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
} }
@ -697,7 +863,10 @@ int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) { if (!existed) {
taosArrayPush(pRemovedTasks, pId); void* p = taosArrayPush(pRemovedTasks, pId);
if (p == NULL) {
mError("failed to put task entry into remove list, taskId:0x%" PRIx64, pId->taskId);
}
} }
} }
@ -759,45 +928,64 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas
} }
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) { while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter); SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask); code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return -1; return code;
} }
} }
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return 0; return code;
} }
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
void *pIter = NULL; void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t)); SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
int32_t code = 0;
mDebug("start to scan checkpoint report info"); mDebug("start to scan checkpoint report info");
while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) {
SArray *pList = *(SArray **)pIter; SArray *pList = *(SArray **)pIter;
STaskChkptInfo* pInfo = taosArrayGet(pList, 0); STaskChkptInfo *pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId); SStreamObj *pStream = NULL;
if (pStream == NULL) { code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId); mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId); void* p = taosArrayPush(pDropped, &pInfo->streamId);
if (p == NULL) {
mError("failed to put stream into drop list:0x%" PRIx64, pInfo->streamId);
}
continue; continue;
} }
int32_t total = mndGetNumOfStreamTasks(pStream); int32_t total = mndGetNumOfStreamTasks(pStream);
int32_t existed = (int32_t) taosArrayGetSize(pList); int32_t existed = (int32_t)taosArrayGetSize(pList);
if (total == existed) { if (total == existed) {
mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info", mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info",
@ -805,17 +993,21 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) { if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId); void* p = taosArrayPush(pDropped, &pInfo->streamId);
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId); if (p == NULL) {
mError("failed to remove stream:0x%" PRIx64, pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
}
} else { } else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet", mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId); pInfo->streamId);
} }
break; break;
} else { } else {
mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); mDebug("stream:0x%" PRIx64 " active checkpoint trans not finished yet, wait", pInfo->streamId);
} }
} else { } else {
mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name, mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name,
@ -829,7 +1021,10 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
if (size > 0) { if (size > 0) {
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i); int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); code = taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
if (code) {
mError("failed to remove stream in buf:0x%"PRIx64, streamId);
}
} }
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
@ -854,29 +1049,30 @@ static int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStream
int32_t blen; int32_t blen;
tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code); tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code);
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
void *pBuf = taosMemoryMalloc(tlen); void *pBuf = taosMemoryMalloc(tlen);
if (pBuf == NULL) { if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen); tEncoderInit(&encoder, abuf, tlen);
tEncodeRestoreCheckpointInfo(&encoder, &req); code = tEncodeRestoreCheckpointInfo(&encoder, &req);
tEncoderClear(&encoder);
if (code == -1) {
taosMemoryFree(pBuf);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)pBuf; SMsgHead *pMsgHead = (SMsgHead *)pBuf;
pMsgHead->contLen = htonl(tlen); pMsgHead->contLen = htonl(tlen);
pMsgHead->vgId = htonl(pTask->info.nodeId); pMsgHead->vgId = htonl(pTask->info.nodeId);
tEncoderClear(&encoder);
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
@ -898,17 +1094,28 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
char msg[128] = {0}; char msg[128] = {0};
snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId); snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId);
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg); STrans *pTrans = NULL;
if (pTrans == NULL) { int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg, &pTrans);
if (pTrans == NULL || code != 0) {
return terrno; return terrno;
} }
STaskId id = {.streamId = pStream->uid, .taskId = taskId}; STaskId id = {.streamId = pStream->uid, .taskId = taskId};
SStreamTask *pTask = mndGetStreamTask(&id, pStream); SStreamTask *pTask = NULL;
ASSERT(pTask); code = mndGetStreamTask(&id, pStream, &pTask);
if (code) {
mError("failed to get task:0x%x in stream:%s, failed to create consensus-checkpointId", taskId, pStream->name);
sdbRelease(pMnode->pSdb, pStream);
return code;
}
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid); code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid);
int32_t code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts); if (code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
}
code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts);
if (code != 0) { if (code != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -916,17 +1123,18 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
} }
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) { if (code) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return code;
} }
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -935,10 +1143,13 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks) { int32_t mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo) {
void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); *pInfo = NULL;
if (pInfo != NULL) {
return (SCheckpointConsensusInfo*)pInfo; void* px = taosHashGet(pHash, &streamId, sizeof(streamId));
if (px != NULL) {
*pInfo = px;
return 0;
} }
SCheckpointConsensusInfo p = { SCheckpointConsensusInfo p = {
@ -947,10 +1158,14 @@ SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId,
.streamId = streamId, .streamId = streamId,
}; };
taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p)); int32_t code = taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
if (code == 0) {
void* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId)); void *pChkptInfo = (SCheckpointConsensusInfo *)taosHashGet(pHash, &streamId, sizeof(streamId));
return pChkptInfo; *pInfo = pChkptInfo;
} else {
*pInfo = NULL;
}
return code;
} }
// no matter existed or not, add the request into info list anyway, since we need to send rsp mannually // no matter existed or not, add the request into info list anyway, since we need to send rsp mannually
@ -971,11 +1186,15 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo
} }
} }
taosArrayPush(pInfo->pTaskList, &info); void *p = taosArrayPush(pInfo->pTaskList, &info);
int32_t num = taosArrayGetSize(pInfo->pTaskList); if (p == NULL) {
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64 mError("s-task:0x%x failed to put task into consensus-checkpointId list, code: out of memory", info.req.taskId);
" waiting tasks:%d", } else {
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num); int32_t num = taosArrayGetSize(pInfo->pTaskList);
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64
" waiting tasks:%d",
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num);
}
} }
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) { void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
@ -984,22 +1203,14 @@ void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
} }
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) { int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {
taosHashRemove(pHash, &streamId, sizeof(streamId)); int32_t code = taosHashRemove(pHash, &streamId, sizeof(streamId));
int32_t numOfStreams = taosHashGetSize(pHash); if (code == 0) {
mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", streamId, int32_t numOfStreams = taosHashGetSize(pHash);
numOfStreams); mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d",
return TSDB_CODE_SUCCESS; streamId, numOfStreams);
} } else {
mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list", streamId);
}
//int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) { return code;
// void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId)); }
// ASSERT(pInfo == NULL);
//
// SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL};
// taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
//
// SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
// ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0);
// mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId);
// return TSDB_CODE_SUCCESS;
//}

View File

@ -155,8 +155,8 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_
SSubmitTbData* pTableData, int64_t earlyTs, const char* id); SSubmitTbData* pTableData, int64_t earlyTs, const char* id);
int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule); SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq);
#define TQ_ERR_GO_TO_END(c) \ #define TQ_ERR_GO_TO_END(c) \
do { \ do { \

View File

@ -23,7 +23,9 @@ static int32_t rsmaRestore(SSma *pSma);
#define SMA_SET_KEEP_CFG(v, l) \ #define SMA_SET_KEEP_CFG(v, l) \
do { \ do { \
SRetention *r = &pCfg->retentions[l]; \ SRetention *r = &pCfg->retentions[l]; \
pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \ int64_t keep = -1; \
convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep); \
pKeepCfg->keep2 = (int32_t)keep; \
pKeepCfg->keep0 = pKeepCfg->keep2; \ pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \ pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \ pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \
@ -60,8 +62,12 @@ static int32_t rsmaRestore(SSma *pSma);
* @return int32_t * @return int32_t
*/ */
static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) { static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) {
int32_t freqDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE); int32_t code = TSDB_CODE_SUCCESS;
int32_t keepDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE);
int64_t freqDuration = -1;
int64_t keepDuration = -1;
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t days = duration; // min int32_t days = duration; // min
if (days < freqDuration) { if (days < freqDuration) {
@ -76,8 +82,8 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
goto _exit; goto _exit;
} }
freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); code = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); code = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level
days *= (nFreqTimes > 1 ? nFreqTimes : 1); days *= (nFreqTimes > 1 ? nFreqTimes : 1);

View File

@ -335,8 +335,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
} }
if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
int64_t msInterval = int64_t msInterval = -1;
convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); TAOS_CHECK_RETURN(convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision,
TIME_UNIT_MILLISECOND, &msInterval));
pItem->maxDelay = (int32_t)msInterval; pItem->maxDelay = (int32_t)msInterval;
} else { } else {
pItem->maxDelay = (int32_t)param->maxdelay[idx]; pItem->maxDelay = (int32_t)param->maxdelay[idx];

View File

@ -69,8 +69,9 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
} }
STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg;
int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); int64_t sInterval = -1;
if (sInterval <= 0) { code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval);
if (TSDB_CODE_SUCCESS != code || 0 == sInterval) {
*days = pTsdbCfg->days; *days = pTsdbCfg->days;
goto _exit; goto _exit;
} }
@ -78,7 +79,11 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
if (records >= SMA_STORAGE_SPLIT_FACTOR) { if (records >= SMA_STORAGE_SPLIT_FACTOR) {
*days = pTsdbCfg->days; *days = pTsdbCfg->days;
} else { } else {
int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); int64_t mInterval = -1;
code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2;
if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { if (daysPerFile > SMA_STORAGE_MINUTES_MAX) {
@ -196,7 +201,12 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE}; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
code = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq);
if (code) {
smaError("failed to build create-table req, code:%d", code);
continue;
}
{ {
uint64_t groupId = pDataBlock->info.id.groupId; uint64_t groupId = pDataBlock->info.id.groupId;

View File

@ -18,8 +18,6 @@
#include "tmsg.h" #include "tmsg.h"
#include "tq.h" #include "tq.h"
#define MAX_CACHE_TABLE_INFO_NUM 10240
typedef struct STableSinkInfo { typedef struct STableSinkInfo {
uint64_t uid; uint64_t uid;
tstr name; tstr name;
@ -43,7 +41,7 @@ static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, con
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
int32_t numOfTags); int32_t numOfTags);
static SArray* createDefaultTagColName(); static int32_t createDefaultTagColName(SArray** pList);
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule); int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo); static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
@ -95,7 +93,10 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
SSingleDeleteReq req = {.startTs = skey, .endTs = ekey}; SSingleDeleteReq req = {.startTs = skey, .endTs = ekey};
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1);
taosArrayPush(deleteReq->deleteReqs, &req); void* p = taosArrayPush(deleteReq->deleteReqs, &req);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
if (originName) name = originName; if (originName) name = originName;
taosMemoryFreeClear(name); taosMemoryFreeClear(name);
@ -149,14 +150,20 @@ static bool tqGetTableInfo(SSHashObj* pTableInfoMap, uint64_t groupId, STableSin
static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
void* buf = NULL; void* buf = NULL;
int32_t tlen = 0; int32_t tlen = 0;
encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
if (code) {
tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code));
return code;
}
SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen}; SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen};
if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg);
if (code) {
tqError("failed to put into write-queue since %s", terrstr()); tqError("failed to put into write-queue since %s", terrstr());
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) { int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
@ -166,18 +173,36 @@ int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const
// set super table name // set super table name
SName name = {0}; SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); int32_t code = tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if (code == 0) {
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code
tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName);
}
}
pCreateTableReq->ctb.tagNum = numOfTags; pCreateTableReq->ctb.tagNum = numOfTags;
return TSDB_CODE_SUCCESS; return code;
} }
SArray* createDefaultTagColName() { int32_t createDefaultTagColName(SArray** pColNameList) {
*pColNameList = NULL;
SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN); SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; if (pTagColNameList == NULL) {
taosArrayPush(pTagColNameList, tagNameStr); return TSDB_CODE_OUT_OF_MEMORY;
return pTagColNameList; }
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
void* p = taosArrayPush(pTagColNameList, tagNameStr);
if (p == NULL) {
taosArrayDestroy(pTagColNameList);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pColNameList = pTagColNameList;
return TSDB_CODE_SUCCESS;
} }
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
@ -201,18 +226,20 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
SStreamTask* pTask, int64_t suid) { SStreamTask* pTask, int64_t suid) {
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId;
tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows); tqDebug("s-task:%s build create %d table(s) msg", id, rows);
int32_t code = 0; int32_t code = 0;
SVCreateTbBatchReq reqs = {0}; SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) { if (NULL == reqs.pArray) {
tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno)); tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno));
goto _end; goto _end;
} }
@ -222,15 +249,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1); int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags); code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
if (code) {
tqError("s-task:%s vgId:%d failed to init create table msg", id, vgId);
continue;
}
taosArrayClear(tagArray); taosArrayClear(tagArray);
if (size == 2) { if (size == 2) {
STagVal tagVal = { STagVal tagVal = {
.cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(tagArray, &tagVal); void* p = taosArrayPush(tagArray, &tagVal);
pCreateTbReq->ctb.tagName = createDefaultTagColName(); if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
if (code) {
return code;
}
} else { } else {
for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
@ -245,14 +283,19 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
} else { } else {
memcpy(&tagVal.i64, pData, pTagData->info.bytes); memcpy(&tagVal.i64, pData, pTagData->info.bytes);
} }
taosArrayPush(tagArray, &tagVal); void* p = taosArrayPush(tagArray, &tagVal);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
} }
} }
tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag); code = tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
taosArrayDestroy(tagArray); taosArrayDestroy(tagArray);
tagArray = NULL; tagArray = NULL;
if (pCreateTbReq->ctb.pTag == NULL) {
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
@ -270,22 +313,34 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid, setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq); void* p = taosArrayPush(reqs.pArray, pCreateTbReq);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
STableSinkInfo* pInfo = NULL; STableSinkInfo* pInfo = NULL;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo); bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo);
if (!alreadyCached) { if (!alreadyCached) {
code = doCreateSinkInfo(pCreateTbReq->name, &pInfo); code = doCreateSinkInfo(pCreateTbReq->name, &pInfo);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr); if (code) {
tqError("vgId:%d failed to create sink tableInfo for table:%s, s-task:%s", vgId, pCreateTbReq->name, id);
continue;
}
code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, id);
if (code) {
tqError("vgId:%d failed to put sink tableInfo:%s into cache, s-task:%s", vgId, pCreateTbReq->name, id);
}
} }
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); tqDebug("s-task:%s build create table:%s msg complete", id, pCreateTbReq->name);
} }
reqs.nReqs = taosArrayGetSize(reqs.pArray); reqs.nReqs = taosArrayGetSize(reqs.pArray);
code = tqPutReqToQueue(pVnode, &reqs); code = tqPutReqToQueue(pVnode, &reqs);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqError("s-task:%s failed to send create table msg", pTask->id.idStr); tqError("s-task:%s failed to send create table msg", id);
} }
_end: _end:
@ -348,15 +403,26 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k); SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k);
if (pNewRow->ts < pOldRow->ts) { if (pNewRow->ts < pOldRow->ts) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1; j += 1;
} else if (pNewRow->ts > pOldRow->ts) { } else if (pNewRow->ts > pOldRow->ts) {
taosArrayPush(pFinal, &pOldRow); void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
} else { } else {
// check for the existance of primary key // check for the existance of primary key
if (pNewRow->numOfPKs == 0) { if (pNewRow->numOfPKs == 0) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
j += 1; j += 1;
tRowDestroy(pOldRow); tRowDestroy(pOldRow);
@ -369,7 +435,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
int32_t ret = tRowKeyCompare(&kNew, &kOld); int32_t ret = tRowKeyCompare(&kNew, &kOld);
if (ret <= 0) { if (ret <= 0) {
taosArrayPush(pFinal, &pNewRow); void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1; j += 1;
if (ret == 0) { if (ret == 0) {
@ -377,7 +447,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
tRowDestroy(pOldRow); tRowDestroy(pOldRow);
} }
} else { } else {
taosArrayPush(pFinal, &pOldRow); void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1; k += 1;
} }
} }
@ -386,12 +460,18 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
while (j < newLen) { while (j < newLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++); SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++);
taosArrayPush(pFinal, &pRow); void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
while (k < oldLen) { while (k < oldLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++); SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++);
taosArrayPush(pFinal, &pRow); void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
taosArrayDestroy(pNew->aRowP); taosArrayDestroy(pNew->aRowP);
@ -425,34 +505,40 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
return true; return true;
} }
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule) { SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq) {
*pReq = NULL;
SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
if (pCreateTbReq == NULL) { if (pCreateTbReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
taosArrayClear(pTagArray); taosArrayClear(pTagArray);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1); int32_t code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
if (code != 0) {
STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; return code;
taosArrayPush(pTagArray, &tagVal);
tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pCreateTbReq->ctb.tagName = createDefaultTagColName(); STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
void* p = taosArrayPush(pTagArray, &tagVal);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
return code;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
// set table name // set table name
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule); setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule);
return pCreateTbReq; *pReq = pCreateTbReq;
return code;
} }
int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) { int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) {
@ -555,7 +641,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
break; break;
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex);
if (colDataIsNull_s(pColData, j)) { if (colDataIsNull_s(pColData, j)) {
@ -566,7 +655,11 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
} }
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
dataIndex++; dataIndex++;
} else { } else {
void* colData = colDataGetData(pColData, j); void* colData = colDataGetData(pColData, j);
@ -574,12 +667,18 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
SValue sv = SValue sv =
(SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else { } else {
SValue sv = {.type = pCol->type}; SValue sv = {.type = pCol->type};
memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv); SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv); void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
dataIndex++; dataIndex++;
} }
@ -596,7 +695,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
} }
ASSERT(pRow); ASSERT(pRow);
taosArrayPush(pTableData->aRowP, &pRow); void* p = taosArrayPush(pTableData->aRowP, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
taosArrayDestroy(pVals); taosArrayDestroy(pVals);
@ -665,6 +767,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
STableSinkInfo* pTableSinkInfo = NULL; STableSinkInfo* pTableSinkInfo = NULL;
int32_t code = 0;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo); bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo);
@ -686,7 +789,11 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} else { // this groupId has not been kept in cache yet } else { // this groupId has not been kept in cache yet
if (dstTableName[0] == 0) { if (dstTableName[0] == 0) {
memset(dstTableName, 0, TSDB_TABLE_NAME_LEN); memset(dstTableName, 0, TSDB_TABLE_NAME_LEN);
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); code = buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
if (code) {
tqDebug("s-task:%s failed to build auto create table-name:%s, groupId:0x%" PRId64, id, dstTableName, groupId);
return code;
}
} else { } else {
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
@ -699,8 +806,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} }
} }
int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo); code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); if (code == 0) {
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName);
} else {
tqDebug("s-task:%s failed to build new sinkTableInfo, dstTable:%s", id, dstTableName);
return code;
}
} }
if (alreadyCached) { if (alreadyCached) {
@ -731,20 +843,20 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = code =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1),
&pTableData->pCreateTbReq);
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) { if (code) {
tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tstrerror(code));
tstrerror(terrno));
taosMemoryFree(pTableSinkInfo); taosMemoryFree(pTableSinkInfo);
return terrno; return code;
} }
pTableSinkInfo->uid = 0; pTableSinkInfo->uid = 0;
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} else { } else {
metaReaderClear(&mr); metaReaderClear(&mr);
@ -765,12 +877,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableSinkInfo->uid = mr.me.uid; pTableSinkInfo->uid = mr.me.uid;
metaReaderClear(&mr); metaReaderClear(&mr);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id); code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} }
} }
} }
return TDB_CODE_SUCCESS; return code;
} }
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
@ -864,14 +976,21 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) { if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq); tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id); (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id);
tbData.pCreateTbReq = NULL; tbData.pCreateTbReq = NULL;
} }
continue; continue;
} }
taosArrayPush(submitReq.aSubmitTbData, &tbData); void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqDebug("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
}
code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1); code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1);
if (code) { // failed and continue
tqDebug("vgId:%d, s-task:%s submit msg failed, data lost", vgId, id);
}
} }
} }
} else { } else {
@ -918,16 +1037,24 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) { if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq); tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id); (void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id);
tbData.pCreateTbReq = NULL; tbData.pCreateTbReq = NULL;
} }
continue; continue;
} }
taosArrayPush(submitReq.aSubmitTbData, &tbData); void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqError("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
continue;
}
int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); code = taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
if (code) {
tqError("vgId:%d, s-task:%s failed to put group into index map, code:%s", vgId, id, tstrerror(code));
continue;
}
} else { } else {
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
@ -951,7 +1078,10 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
taosHashCleanup(pTableIndexMap); taosHashCleanup(pTableIndexMap);
if (hasSubmit) { if (hasSubmit) {
doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks); code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks);
if (code) { // failed and continue
tqError("vgId:%d failed to build and send submit msg", vgId);
}
} else { } else {
tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
tqDebug("vgId:%d, s-task:%s write results completed", vgId, id); tqDebug("vgId:%d, s-task:%s write results completed", vgId, id);
@ -989,7 +1119,11 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char
} }
int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId)); int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId));
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId); if (code == 0) {
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId);
} else {
tqError("s-task:%s failed to remove table meta from hashmap, groupId:%" PRId64, id, groupId);
}
return code; return code;
} }
@ -1019,10 +1153,14 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock*
void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
tEncoderInit(&encoder, abuf, len); tEncoderInit(&encoder, abuf, len);
tEncodeSBatchDeleteReq(&encoder, &deleteReq); code = tEncodeSBatchDeleteReq(&encoder, &deleteReq);
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosArrayDestroy(deleteReq.deleteReqs); taosArrayDestroy(deleteReq.deleteReqs);
if (code) {
return code;
}
((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode); ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode);
SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)}; SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)};

View File

@ -37,7 +37,12 @@ int32_t tqScanWal(STQ* pTq) {
// check all tasks // check all tasks
int32_t numOfTasks = 0; int32_t numOfTasks = 0;
bool shouldIdle = true; bool shouldIdle = true;
doScanWalForAllTasks(pMeta, &shouldIdle);
int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle);
if (code) {
tqError("vgId:%d failed to start all tasks, try next time", vgId);
return code;
}
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
int32_t times = (--pMeta->scanInfo.scanCounter); int32_t times = (--pMeta->scanInfo.scanCounter);
@ -51,9 +56,13 @@ int32_t tqScanWal(STQ* pTq) {
if (times > 0) { if (times > 0) {
tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION); tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION);
tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION); code = tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION);
if (code) {
tqError("vgId:%d sched scan wal in %dms failed, ignore this failure", vgId, SCAN_WAL_IDLE_DURATION);
}
} }
return 0;
return code;
} }
typedef struct SBuildScanWalMsgParam { typedef struct SBuildScanWalMsgParam {
@ -69,28 +78,44 @@ static void doStartScanWal(void* param, void* tmrId) {
tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks, tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks,
pTq->pVnode->restored); pTq->pVnode->restored);
/*int32_t code = */ streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA); int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA);
taosMemoryFree(pParam); taosMemoryFree(pParam);
if (code) {
tqError("vgId:%d failed sched task to scan wal", vgId);
}
} }
int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) { int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam)); SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam));
if (pParam == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pParam->pTq = pTq; pParam->pTq = pTq;
pParam->numOfTasks = numOfTasks; pParam->numOfTasks = numOfTasks;
tmr_h pTimer = streamTimerGetInstance(); tmr_h pTimer = NULL;
ASSERT(pTimer); code = streamTimerGetInstance(&pTimer);
if (code) {
tqError("vgId:%d failed to get tmr ctrl during sched scan wal", vgId);
return code;
}
if (pMeta->scanInfo.scanTimer == NULL) { if (pMeta->scanInfo.scanTimer == NULL) {
pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer); pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer);
} else { } else {
taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer); code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer);
if (code) {
tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration);
}
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
@ -207,7 +232,11 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) {
double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0; double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0;
qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs", qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs",
id, pTask->step2Range.minVer, maxVer, el); id, pTask->step2Range.minVer, maxVer, el);
/*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask); int32_t code = streamTaskPutTranstateIntoInputQ(pTask);
if (code) {
qError("s-task:%s failed to put trans-state into inputQ", id);
}
return true; return true;
} else { } else {
qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64 qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64
@ -290,8 +319,12 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems
break; break;
} }
} else { } else {
walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer);
code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
if (code) {
tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer);
}
break; break;
} }
} }
@ -347,18 +380,18 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX; int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX;
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__READY) { if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name); tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;
} }
bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems); bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if ((numOfItems > 0) || hasNewData) { if ((numOfItems > 0) || hasNewData) {
noDataInWal = false; noDataInWal = false;
@ -366,7 +399,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
taosArrayDestroy(pTaskList); taosArrayDestroy(pTaskList);
return -1; return code;
} }
} }
@ -379,5 +412,5 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
} }
taosArrayDestroy(pTaskList); taosArrayDestroy(pTaskList);
return 0; return TSDB_CODE_SUCCESS;
} }

View File

@ -160,6 +160,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
bool updated = false; bool updated = false;
int32_t code = 0;
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
@ -258,26 +259,40 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
} }
} }
// save // stream do update the nodeEp info, write it into stream meta.
if (updated) { if (updated) {
tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId); tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId);
streamMetaSaveTask(pMeta, pTask); code = streamMetaSaveTask(pMeta, pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamMetaSaveTask(pMeta, *ppHTask); code = streamMetaSaveTask(pMeta, *ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code));
}
} }
} else { } else {
tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId); tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId);
} }
streamTaskStop(pTask); code = streamTaskStop(pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamTaskStop(*ppHTask); code = streamTaskStop(*ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code));
}
} }
// keep info // keep info
streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st); streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st);
rsp.code = 0; rsp.code = TSDB_CODE_SUCCESS;
// possibly only handle the stream task. // possibly only handle the stream task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
@ -305,13 +320,16 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
#if 0 #if 0
taosMSleep(5000);// for test purpose, to trigger the leader election taosMSleep(5000);// for test purpose, to trigger the leader election
#endif #endif
tqStreamTaskStartAsync(pMeta, cb, true); code = tqStreamTaskStartAsync(pMeta, cb, true);
if (code) {
tqError("vgId:%d async start all tasks, failed, code:%s", vgId, tstrerror(code));
}
} }
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList); taosArrayDestroy(req.pNodeList);
return rsp.code; return rsp.code; // always return true
} }
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -333,7 +351,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask) { if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0}; SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) { if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
return -1; return -1;
@ -393,14 +411,14 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask) { if (pTask && (code == 0)) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code); code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} else { } else {
tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId); tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId);
terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST; terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST;
return terrno; return TSDB_CODE_STREAM_TASK_NOT_EXIST;
} }
} }
@ -408,16 +426,22 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SDecoder decoder; SDecoder decoder;
SStreamRetrieveReq req; SStreamRetrieveReq req;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req); code = tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d failed to decode retrieve msg, quit handling it", pMeta->vgId);
return code;
}
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || code != 0) {
tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
req.dstTaskId); req.dstTaskId);
tCleanupStreamRetrieveReq(&req); tCleanupStreamRetrieveReq(&req);
@ -446,6 +470,7 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont; char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamTaskCheckReq req; SStreamTaskCheckReq req;
SStreamTaskCheckRsp rsp = {0}; SStreamTaskCheckRsp rsp = {0};
@ -453,9 +478,14 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamTaskCheckReq(&decoder, &req); code = tDecodeStreamTaskCheckReq(&decoder, &req);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d decode check msg failed, not handle this msg", pMeta->vgId);
return code;
}
streamTaskProcessCheckMsg(pMeta, &req, &rsp); streamTaskProcessCheckMsg(pMeta, &req, &rsp);
return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId); return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId);
} }
@ -490,7 +520,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask); code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask);
if (pTask == NULL) { if ((pTask == NULL) || (code != 0)) {
return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId); return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId);
} }
@ -518,19 +548,25 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask);
if (pTask == NULL) { if (code != 0) {
tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId,
pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
if (code) {
return code;
}
{ // send checkpoint ready rsp { // send checkpoint ready rsp
SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg)); SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg));
if (pReadyRsp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReadyRsp->upstreamTaskId = req.upstreamTaskId; pReadyRsp->upstreamTaskId = req.upstreamTaskId;
pReadyRsp->upstreamNodeId = req.upstreamNodeId; pReadyRsp->upstreamNodeId = req.upstreamNodeId;
@ -606,8 +642,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
if (restored) { if (restored) {
SStreamTask* p = NULL; SStreamTask* p = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &p); code = streamMetaAcquireTask(pMeta, streamId, taskId, &p);
if ((p != NULL) && (p->info.fillHistory == 0)) { if ((p != NULL) && (code == 0) && (p->info.fillHistory == 0)) {
tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId); code = tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
} }
if (p != NULL) { if (p != NULL) {
@ -631,6 +667,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
STaskId hTaskId = {0}; STaskId hTaskId = {0};
tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
@ -649,8 +686,12 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamTaskSetRemoveBackendFiles(pTask); streamTaskSetRemoveBackendFiles(pTask);
streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
if (code) {
tqError("s-task:0x%x failed to clear related fill-history info, still exists", pReq->taskId);
}
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -658,11 +699,17 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
// drop the related fill-history task firstly // drop the related fill-history task firstly
if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
}
} }
// drop the stream task now // drop the stream task now
streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId); code = streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop task failed", pReq->taskId, vgId);
}
// commit the update // commit the update
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
@ -674,12 +721,13 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; return 0; // always return success
} }
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) { int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId); tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId);
@ -689,7 +737,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask != NULL && (*ppTask) != NULL) { if (ppTask != NULL && (*ppTask) != NULL) {
streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq); code = streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq);
} else { // failed to get the task. } else { // failed to get the task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
tqError( tqError(
@ -700,7 +748,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// always return success when handling the requirement issued by mnode during transaction. // always return success when handling the requirement issued by mnode during transaction.
return TSDB_CODE_SUCCESS; return code;
} }
static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
@ -746,7 +794,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
if (isLeader && !tsDisableStream) { if (isLeader && !tsDisableStream) {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
streamMetaStartAllTasks(pMeta); code = streamMetaStartAllTasks(pMeta);
} else { } else {
streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId); streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId);
pMeta->startInfo.restartCount = 0; pMeta->startInfo.restartCount = 0;
@ -765,16 +813,16 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) { if (type == STREAM_EXEC_T_START_ONE_TASK) {
streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); (void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0; return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) { } else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
streamMetaStartAllTasks(pMeta); (void) streamMetaStartAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
restartStreamTasks(pMeta, isLeader); (void) restartStreamTasks(pMeta, isLeader);
return 0; return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) { } else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
streamMetaStopAllTasks(pMeta); (void) streamMetaStopAllTasks(pMeta);
return 0; return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) { } else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId); int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
@ -783,7 +831,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { if (pTask != NULL && (code == 0)) {
char* pStatus = NULL; char* pStatus = NULL;
if (streamTaskReadyToRun(pTask, &pStatus)) { if (streamTaskReadyToRun(pTask, &pStatus)) {
int64_t execTs = pTask->status.lastExecTs; int64_t execTs = pTask->status.lastExecTs;
@ -804,12 +852,12 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { // even in halt status, the data in inputQ must be processed if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL; char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) { if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer); pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
streamExecTask(pTask); (void) streamExecTask(pTask);
} else { } else {
int8_t status = streamTaskSetSchedStatusInactive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -829,6 +877,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskStartInfo* pStartInfo = &pMeta->startInfo;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
bool scanWal = false; bool scanWal = false;
int32_t code = 0;
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
if (pStartInfo->startAllTasks == 1) { if (pStartInfo->startAllTasks == 1) {
@ -844,8 +893,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
pStartInfo->restartCount); pStartInfo->restartCount);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); return restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER));
return TSDB_CODE_SUCCESS;
} else { } else {
if (pStartInfo->restartCount == 0) { if (pStartInfo->restartCount == 0) {
tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId); tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId);
@ -862,10 +910,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
if (scanWal && (vgId != SNODE_HANDLE)) { if (scanWal && (vgId != SNODE_HANDLE)) {
tqDebug("vgId:%d start scan wal for executing tasks", vgId); tqDebug("vgId:%d start scan wal for executing tasks", vgId);
tqScanWalAsync(pMeta->ahandle, true); code = tqScanWalAsync(pMeta->ahandle, true);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
@ -873,7 +921,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId); pMeta->vgId, pReq->taskId);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -881,7 +929,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
streamTaskClearCheckInfo(pTask, true); streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks // clear flag set during do checkpoint, and open inputQ for all upstream tasks
@ -904,7 +952,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name); tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
} }
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -915,7 +963,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId); pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
@ -929,11 +977,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
pTask->id.idStr, (int32_t)pReq->downstreamTaskId); pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS); TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return code;
return TSDB_CODE_SUCCESS;
} }
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
@ -948,7 +995,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
// re-send the lost checkpoint-trigger msg to downstream task // re-send the lost checkpoint-trigger msg to downstream task
tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
(int32_t)pReq->downstreamTaskId, checkpointId, transId); (int32_t)pReq->downstreamTaskId, checkpointId, transId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_SUCCESS); TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait } else { // not send checkpoint-trigger yet, wait
int32_t recv = 0, total = 0; int32_t recv = 0, total = 0;
@ -962,7 +1009,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"sending checkpoint-source/trigger", "sending checkpoint-source/trigger",
pTask->id.idStr, recv, total); pTask->id.idStr, recv, total);
} }
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
} else { // upstream not recv the checkpoint-source/trigger till now } else { // upstream not recv the checkpoint-source/trigger till now
@ -971,12 +1018,12 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger", "upstream sending checkpoint-source/trigger",
pTask->id.idStr); pTask->id.idStr);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS); TSDB_CODE_ACTION_IN_PROGRESS);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -984,19 +1031,19 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError( tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pRsp->taskId); pMeta->vgId, pRsp->taskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d", tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
@ -1004,7 +1051,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId, tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId); pReq->taskId);
// since task is in [STOP|DROPPING] state, it is safe to assume the pause is active // since task is in [STOP|DROPPING] state, it is safe to assume the pause is active
@ -1018,7 +1065,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pHistoryTask = NULL; pHistoryTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask); code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask);
if (pHistoryTask == NULL) { if (pHistoryTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
", it may have been dropped already", ", it may have been dropped already",
pMeta->vgId, pTask->hTaskInfo.id.taskId); pMeta->vgId, pTask->hTaskInfo.id.taskId);
@ -1042,6 +1089,8 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
bool fromVnode) { bool fromVnode) {
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
int32_t code = 0;
if (pTask == NULL) { if (pTask == NULL) {
return -1; return -1;
} }
@ -1065,18 +1114,18 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) { if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
pTask->hTaskInfo.operatorOpen = false; pTask->hTaskInfo.operatorOpen = false;
streamStartScanHistoryAsync(pTask, igUntreated); code = streamStartScanHistoryAsync(pTask, igUntreated);
} else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) { } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
tqScanWalAsync((STQ*)handle, false); code = tqScanWalAsync((STQ*)handle, false);
} else { } else {
streamTrySchedExec(pTask); code = streamTrySchedExec(pTask);
} }
} /*else { } /*else {
ASSERT(status != TASK_STATUS__UNINIT); ASSERT(status != TASK_STATUS__UNINIT);
}*/ }*/
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return 0; return code;
} }
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) { int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) {
@ -1086,15 +1135,15 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId); tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask); SStreamTaskState pState = streamTaskGetStatus(pTask);
tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name); tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
if (code != 0) { if (code != 0) {
@ -1104,11 +1153,11 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL; SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask); code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask) { if (pHTask && (code == 0)) {
taosThreadMutexLock(&pHTask->lock); streamMutexLock(&pHTask->lock);
SStreamTaskState p = streamTaskGetStatus(pHTask); SStreamTaskState p = streamTaskGetStatus(pHTask);
tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name); tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name);
taosThreadMutexUnlock(&pHTask->lock); streamMutexUnlock(&pHTask->lock);
code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode); code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode);
} }
@ -1139,15 +1188,15 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask); int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId); pRsp->downstreamNodeId, pRsp->downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST; return code;
} }
streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); code = streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }
int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -1173,11 +1222,11 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL; SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask); code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already", tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId); pMeta->vgId, req.taskId);
streamMetaAddFailedTask(pMeta, req.streamId, req.taskId); (void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return TSDB_CODE_SUCCESS; return code;
} }
// discard the rsp, since it is expired. // discard the rsp, since it is expired.
@ -1193,13 +1242,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode", tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode",
pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId); pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId);
taosThreadMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId); ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) { if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard", tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId); pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1215,14 +1264,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
} }
pTask->chkInfo.consensusTransId = req.transId; pTask->chkInfo.consensusTransId = req.transId;
taosThreadMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
if (pMeta->role == NODE_ROLE_LEADER) { if (pMeta->role == NODE_ROLE_LEADER) {
/*code = */ tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId); code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId);
} else { } else {
tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr); tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return code;
} }

View File

@ -176,7 +176,14 @@ typedef struct SExplainCtx {
#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join") #define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
#define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column")) #define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column"))
#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u))) #define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p, _r) \
do { \
if ((_u) == 'n' || (_u) == 'y') { \
_r = (_t); \
} else { \
code = convertTimeFromPrecisionToUnit(_t, _p, _u, &_r); \
} \
} while(0)
#define EXPLAIN_ROW_NEW(level, ...) \ #define EXPLAIN_ROW_NEW(level, ...) \
do { \ do { \

View File

@ -55,7 +55,7 @@ char* qExplainGetAsofOpStr(int32_t opType) {
case OP_TYPE_LOWER_EQUAL: case OP_TYPE_LOWER_EQUAL:
return "<="; return "<=";
case OP_TYPE_EQUAL: case OP_TYPE_EQUAL:
return "="; return "=";
default: default:
return "UNKNOWN"; return "UNKNOWN";
} }
@ -192,13 +192,13 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S
if (0 == group->physiPlanExecIdx) { if (0 == group->physiPlanExecIdx) {
group->nodeIdx = 0; group->nodeIdx = 0;
} }
rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++);
if (group->physiPlanExecIdx >= rsp->numOfPlans) { if (group->physiPlanExecIdx >= rsp->numOfPlans) {
qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno; if(taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx) == NULL) return terrno;
} else { } else {
for (int32_t i = 0; i < group->nodeNum; ++i) { for (int32_t i = 0; i < group->nodeNum; ++i) {
@ -338,7 +338,7 @@ static char* qExplainGetScanMode(STableScanPhysiNode* pScan) {
isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname; isGroupByTag = (NULL != pScan->pGroupTags) && !isGroupByTbname;
if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { if ((((!isGroupByTag) || isGroupByTbname) && pScan->groupSort) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) {
return "seq_grp_order"; return "seq_grp_order";
} }
if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) { if ((isGroupByTbname && (pScan->groupSort || pScan->scan.groupOrderScan)) || (isGroupByTag && (pScan->groupSort || pScan->scan.groupOrderScan))) {
return "grp_order"; return "grp_order";
@ -416,7 +416,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -581,7 +581,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -621,7 +621,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False"); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGEBLOCKS_FORMAT, pPrjNode->mergeDataBlock? "True":"False");
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pPrjNode->node.pConditions) { if (pPrjNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
@ -629,7 +629,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -689,7 +689,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin); EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
@ -705,7 +705,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) { if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) {
@ -733,7 +733,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
if (NULL != pJoinNode->pColOnCond) { if (NULL != pJoinNode->pColOnCond) {
@ -741,7 +741,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
@ -831,7 +831,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -844,7 +844,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId);
QRY_ERR_RET(TSDB_CODE_APP_ERROR); QRY_ERR_RET(TSDB_CODE_APP_ERROR);
} }
nodeNum += group->nodeNum; nodeNum += group->nodeNum;
} }
@ -876,7 +876,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) { for (int32_t i = pExchNode->srcStartGroupId; i <= pExchNode->srcEndGroupId; ++i) {
@ -956,7 +956,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -991,10 +991,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1043,10 +1050,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1123,7 +1137,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1163,7 +1177,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1205,7 +1219,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1247,7 +1261,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1340,7 +1354,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1381,7 +1395,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1422,7 +1436,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1464,7 +1478,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1535,7 +1549,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1563,10 +1577,17 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
uint8_t precision = qExplainGetIntervalPrecision(pIntNode); uint8_t precision = qExplainGetIntervalPrecision(pIntNode);
int64_t time1 = -1;
int64_t time2 = -1;
int32_t code = TSDB_CODE_SUCCESS;
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision, time1);
QRY_ERR_RET(code);
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision, time2);
QRY_ERR_RET(code);
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT,
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision), time1,
pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision),
INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), time2,
pIntNode->slidingUnit); pIntNode->slidingUnit);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@ -1577,7 +1598,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1644,7 +1665,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
} }
@ -1669,7 +1690,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE, QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
@ -1722,7 +1743,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
if (pJoinNode->pPrimKeyCond) { if (pJoinNode->pPrimKeyCond) {
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pColEqCond) { if (pJoinNode->pColEqCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1730,7 +1751,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pColEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pTagEqCond) { if (pJoinNode->pTagEqCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1738,7 +1759,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET( QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true; conditionsGot = true;
} }
if (pJoinNode->pFullOnCond) { if (pJoinNode->pFullOnCond) {
if (conditionsGot) { if (conditionsGot) {
@ -1746,14 +1767,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
} }
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
} }
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->timeRangeTarget) { if (pJoinNode->timeRangeTarget) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey); EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey);
EXPLAIN_ROW_END(); EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
} }
} }
break; break;
@ -2124,7 +2145,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_ERRNO(code); TAOS_CHECK_ERRNO(code);
} }
groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum); groupDone = (taosArrayGetSize(group->nodeExecInfo) >= group->nodeNum);
taosWUnLockLatch(&group->lock); taosWUnLockLatch(&group->lock);

View File

@ -482,7 +482,8 @@ void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
} }
len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true); void* pTmpBuf = pBuf;
len = doStreamCountEncodeOpState(&pTmpBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len); strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), pBuf, len);
saveStreamOperatorStateComplete(&pInfo->basic); saveStreamOperatorStateComplete(&pInfo->basic);

View File

@ -542,7 +542,11 @@ void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator); int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamEventEncodeOpState(&pBuf, len, pOperator); len = doStreamEventEncodeOpState(&pBuf, len, pOperator);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);

View File

@ -1410,7 +1410,11 @@ void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator); int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator); len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME, pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
@ -3193,7 +3197,11 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true); int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true); len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
@ -4401,7 +4409,11 @@ void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true); int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
void* buf = taosMemoryCalloc(1, len); void* buf = taosMemoryCalloc(1, len);
void* pBuf = buf; if (!buf) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return;
}
void* pBuf = buf;
len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true); len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len); strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);

View File

@ -180,7 +180,7 @@ int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnI
goto _exit; goto _exit;
} }
colDataSetVal(pOutputData, i, output, (output == NULL)); code = colDataSetVal(pOutputData, i, output, (output == NULL));
_exit: _exit:
if (output) { if (output) {
@ -200,7 +200,7 @@ int32_t executeAsTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoDat
goto _exit; goto _exit;
} }
colDataSetVal(pOutputData, i, output, (output == NULL)); code = colDataSetVal(pOutputData, i, output, (output == NULL));
_exit: _exit:
if (output) { if (output) {
@ -228,7 +228,7 @@ int32_t executeRelationFunc(const GEOSGeometry *geom1, const GEOSPreparedGeometr
} }
} }
colDataSetVal(pOutputData, i, &res, (res==-1)); code = colDataSetVal(pOutputData, i, &res, (res==-1));
return code; return code;
} }

View File

@ -86,9 +86,9 @@ _exit:
return code; return code;
} }
static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) { static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
int ret = 0; int32_t code = 0;
char *wktPatternWithSpace = taosMemoryCalloc(4, 1024); char *wktPatternWithSpace = taosMemoryCalloc(4, 1024);
sprintf( sprintf(
wktPatternWithSpace, wktPatternWithSpace,
"^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( " "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
@ -142,9 +142,9 @@ static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
"*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( " "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( "
"*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$"); "*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$");
ret = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace); code = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace);
taosMemoryFree(wktPatternWithSpace); taosMemoryFree(wktPatternWithSpace);
return ret; return code;
} }
int32_t initCtxGeomFromText() { int32_t initCtxGeomFromText() {

View File

@ -1523,7 +1523,7 @@ static int32_t biMakeTbnameProjectAstNode(char* funcName, char* tableAlias, SNod
if (!multiResFunc) { if (!multiResFunc) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN);
code = nodesListMakeStrictAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); code = nodesListMakeStrictAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc);
@ -5233,7 +5233,8 @@ static int32_t translateFill(STranslateContext* pCxt, SSelectStmt* pSelect, SInt
} }
static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) {
int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); int64_t days = -1;
convertTimeFromPrecisionToUnit(val, fromPrecision, 'd', &days);
switch (unit) { switch (unit) {
case 'b': case 'b':
case 'u': case 'u':
@ -8232,7 +8233,8 @@ static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) {
if (NULL == pVal) { if (NULL == pVal) {
return NULL; return NULL;
} }
int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit); int64_t timeVal = -1;
convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit, &timeVal);
char buf[20] = {0}; char buf[20] = {0};
int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit);
pVal->literal = strndup(buf, len); pVal->literal = strndup(buf, len);
@ -8781,7 +8783,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = tNameExtractFullName(&name, usedbReq.db); code = tNameExtractFullName(&name, usedbReq.db);
} }
if (TSDB_CODE_SUCCESS == code) if (TSDB_CODE_SUCCESS == code)
code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs); code = getDBVgVersion(pCxt, usedbReq.db, &usedbReq.vgVersion, &usedbReq.dbId, &usedbReq.numOfTable, &usedbReq.stateTs);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq); code = buildCmdMsg(pCxt, TDMT_MND_USE_DB, (FSerializeFunc)tSerializeSUseDbReq, &usedbReq);

View File

@ -231,14 +231,8 @@ void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t up
int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId,
int64_t checkpointId, SRpcMsg* pMsg); int64_t checkpointId, SRpcMsg* pMsg);
typedef int32_t (*__stream_async_exec_fn_t)(void* param);
int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code);
void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -1540,7 +1540,7 @@ int32_t chkpLoadExtraInfo(char* pChkpIdDir, int64_t* chkpId, int64_t* processId)
// compatible with previous version // compatible with previous version
*processId = -1; *processId = -1;
code = 0; code = 0;
stError("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(TAOS_SYSTEM_ERROR(errno))); stWarn("failed to open file to load extra info, file:%s, reason:%s", pDst, tstrerror(TAOS_SYSTEM_ERROR(errno)));
goto _EXIT; goto _EXIT;
} }
@ -2308,6 +2308,7 @@ _EXIT:
taosMemoryFree(cfHandle); taosMemoryFree(cfHandle);
return code; return code;
} }
void* taskDbAddRef(void* pTaskDb) { void* taskDbAddRef(void* pTaskDb) {
STaskDbWrapper* pBackend = pTaskDb; STaskDbWrapper* pBackend = pTaskDb;
return taosAcquireRef(taskDbWrapperId, pBackend->refId); return taosAcquireRef(taskDbWrapperId, pBackend->refId);

View File

@ -45,7 +45,7 @@ typedef struct STaskInitTs {
SMetaRefMgt gMetaRefMgt; SMetaRefMgt gMetaRefMgt;
void metaRefMgtInit(); int32_t metaRefMgtInit();
void metaRefMgtCleanup(); void metaRefMgtCleanup();
int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid); int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid);
@ -56,9 +56,14 @@ static void streamMetaEnvInit() {
streamMetaId = taosOpenRef(64, streamMetaCloseImpl); streamMetaId = taosOpenRef(64, streamMetaCloseImpl);
metaRefMgtInit(); int32_t code = metaRefMgtInit();
int32_t code = streamTimerInit(); if (code) {
if (code != 0) { stError("failed to init stream meta mgmt env, start failed");
return;
}
code = streamTimerInit();
if (code) {
stError("failed to init stream meta env, start failed"); stError("failed to init stream meta env, start failed");
} }
} }
@ -66,17 +71,29 @@ static void streamMetaEnvInit() {
void streamMetaInit() { (void) taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } void streamMetaInit() { (void) taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); }
void streamMetaCleanup() { void streamMetaCleanup() {
taosCloseRef(streamBackendId); (void) taosCloseRef(streamBackendId);
taosCloseRef(streamBackendCfWrapperId); (void) taosCloseRef(streamBackendCfWrapperId);
taosCloseRef(streamMetaId); (void) taosCloseRef(streamMetaId);
metaRefMgtCleanup(); metaRefMgtCleanup();
streamTimerCleanUp(); streamTimerCleanUp();
} }
void metaRefMgtInit() { int32_t metaRefMgtInit() {
taosThreadMutexInit(&(gMetaRefMgt.mutex), NULL); int32_t code = taosThreadMutexInit(&(gMetaRefMgt.mutex), NULL);
gMetaRefMgt.pTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (code) {
return code;
}
if (code == 0) {
gMetaRefMgt.pTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
}
if (gMetaRefMgt.pTable == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} else {
return code;
}
} }
void metaRefMgtCleanup() { void metaRefMgtCleanup() {
@ -96,20 +113,34 @@ void metaRefMgtCleanup() {
} }
int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) { int32_t metaRefMgtAdd(int64_t vgId, int64_t* rid) {
int32_t code = 0;
void* p = NULL;
streamMutexLock(&gMetaRefMgt.mutex); streamMutexLock(&gMetaRefMgt.mutex);
void* p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId)); p = taosHashGet(gMetaRefMgt.pTable, &vgId, sizeof(vgId));
if (p == NULL) { if (p == NULL) {
SArray* list = taosArrayInit(8, sizeof(void*)); SArray* list = taosArrayInit(8, sizeof(void*));
taosArrayPush(list, &rid); p = taosArrayPush(list, &rid);
taosHashPut(gMetaRefMgt.pTable, &vgId, sizeof(vgId), &list, sizeof(void*)); if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = taosHashPut(gMetaRefMgt.pTable, &vgId, sizeof(vgId), &list, sizeof(void*));
if (code) {
stError("vgId:%d failed to put into metaRef table, rid:%" PRId64, (int32_t) vgId, *rid);
return code;
}
} else { } else {
SArray* list = *(SArray**)p; SArray* list = *(SArray**)p;
taosArrayPush(list, &rid); void* px = taosArrayPush(list, &rid);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
} }
streamMutexUnlock(&gMetaRefMgt.mutex); streamMutexUnlock(&gMetaRefMgt.mutex);
return 0; return code;
} }
int32_t streamMetaOpenTdb(SStreamMeta* pMeta) { int32_t streamMetaOpenTdb(SStreamMeta* pMeta) {
@ -141,19 +172,25 @@ enum STREAM_STATE_VER {
}; };
int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) {
int8_t ret = STREAM_STATA_COMPATIBLE; int8_t ret = STREAM_STATA_COMPATIBLE;
TBC* pCur = NULL; TBC* pCur = NULL;
int32_t code = 0;
if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream
return ret;
}
void* pKey = NULL; void* pKey = NULL;
int32_t kLen = 0; int32_t kLen = 0;
void* pVal = NULL; void* pVal = NULL;
int32_t vLen = 0; int32_t vLen = 0;
tdbTbcMoveToFirst(pCur); if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream
return ret;
}
code = tdbTbcMoveToFirst(pCur);
if (code) {
(void) tdbTbcClose(pCur);
stError("vgId:%d failed to open stream meta file cursor, not perform compatible check", pMeta->vgId);
return ret;
}
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
if (pVal == NULL || vLen == 0) { if (pVal == NULL || vLen == 0) {
break; break;
@ -178,7 +215,7 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) {
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
tdbTbcClose(pCur); (void) tdbTbcClose(pCur);
return ret; return ret;
} }
@ -244,7 +281,11 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
streamMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key));
if ((ppBackend != NULL) && (*ppBackend != NULL)) { if ((ppBackend != NULL) && (*ppBackend != NULL)) {
taskDbAddRef(*ppBackend); void* p = taskDbAddRef(*ppBackend);
if (p == NULL) {
stError("s-task:0x%x failed to ref backend", pTask->id.taskId);
return TSDB_CODE_FAILED;
}
STaskDbWrapper* pBackend = *ppBackend; STaskDbWrapper* pBackend = *ppBackend;
pBackend->pMeta = pMeta; pBackend->pMeta = pMeta;
@ -278,7 +319,10 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key)
if (processVer != -1) pTask->chkInfo.processedVer = processVer; if (processVer != -1) pTask->chkInfo.processedVer = processVer;
taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*)); int32_t code = taosHashPut(pMeta->pTaskDbUnique, key, strlen(key), &pBackend, sizeof(void*));
if (code) {
stError("s-task:0x%x failed to put taskDb backend, code:out of memory", pTask->id.taskId);
}
streamMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend);
@ -290,7 +334,10 @@ void streamMetaRemoveDB(void* arg, char* key) {
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
streamMutexLock(&pMeta->backendMutex); streamMutexLock(&pMeta->backendMutex);
taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key)); int32_t code = taosHashRemove(pMeta->pTaskDbUnique, key, strlen(key));
if (code) {
stError("vgId:%d failed to remove key:%s in taskDbUnique map", pMeta->vgId, key);
}
streamMutexUnlock(&pMeta->backendMutex); streamMutexUnlock(&pMeta->backendMutex);
} }
@ -398,12 +445,22 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn,
} }
#endif #endif
taosThreadRwlockInit(&pMeta->lock, &attr); code = taosThreadRwlockInit(&pMeta->lock, &attr);
taosThreadRwlockAttrDestroy(&attr); if (code) {
goto _err;
}
code = taosThreadRwlockAttrDestroy(&attr);
if (code) {
goto _err;
}
int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t));
memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid));
metaRefMgtAdd(pMeta->vgId, pRid); code = metaRefMgtAdd(pMeta->vgId, pRid);
if (code) {
goto _err;
}
code = createMetaHbInfo(pRid, &pMeta->pHbInfo); code = createMetaHbInfo(pRid, &pMeta->pHbInfo);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -416,7 +473,8 @@ int32_t streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn,
if (pMeta->bkdChkptMgt == NULL) { if (pMeta->bkdChkptMgt == NULL) {
goto _err; goto _err;
} }
taosThreadMutexInit(&pMeta->backendMutex, NULL);
code = taosThreadMutexInit(&pMeta->backendMutex, NULL);
*p = pMeta; *p = pMeta;
return code; return code;
@ -425,9 +483,9 @@ _err:
taosMemoryFree(pMeta->path); taosMemoryFree(pMeta->path);
if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap); if (pMeta->pTasksMap) taosHashCleanup(pMeta->pTasksMap);
if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList);
if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); if (pMeta->pTaskDb) (void)tdbTbClose(pMeta->pTaskDb);
if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); if (pMeta->pCheckpointDb) (void)tdbTbClose(pMeta->pCheckpointDb);
if (pMeta->db) tdbClose(pMeta->db); if (pMeta->db) (void) tdbClose(pMeta->db);
if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo); if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo);
if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks); if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks);
if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet); if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet);
@ -473,7 +531,7 @@ void streamMetaClear(SStreamMeta* pMeta) {
// release the ref by timer // release the ref by timer
if (p->info.delaySchedParam != 0 && p->info.fillHistory == 0) { // one more ref in timer if (p->info.delaySchedParam != 0 && p->info.fillHistory == 0) { // one more ref in timer
stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt); stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt);
taosTmrStop(p->schedInfo.pDelayTimer); (void) taosTmrStop(p->schedInfo.pDelayTimer);
p->info.delaySchedParam = 0; p->info.delaySchedParam = 0;
streamMetaReleaseTask(pMeta, p); streamMetaReleaseTask(pMeta, p);
} }
@ -481,7 +539,11 @@ void streamMetaClear(SStreamMeta* pMeta) {
streamMetaReleaseTask(pMeta, p); streamMetaReleaseTask(pMeta, p);
} }
taosRemoveRef(streamBackendId, pMeta->streamBackendRid); int32_t code = taosRemoveRef(streamBackendId, pMeta->streamBackendRid);
if (code) {
stError("vgId:%d remove stream backend Ref failed, rid:%"PRId64, pMeta->vgId, pMeta->streamBackendRid);
}
taosHashClear(pMeta->pTasksMap); taosHashClear(pMeta->pTasksMap);
taosArrayClear(pMeta->pTaskList); taosArrayClear(pMeta->pTaskList);
@ -502,14 +564,7 @@ void streamMetaClose(SStreamMeta* pMeta) {
if (pMeta == NULL) { if (pMeta == NULL) {
return; return;
} }
(void) taosRemoveRef(streamMetaId, pMeta->rid);
// int64_t rid = *(int64_t*)pMeta->pRid;
// if (taosTmrStop(pMeta->hbInfo.hbTmr)) {
// taosMemoryFree(pMeta->pRid);
// } else {
// // do nothing, stop by timer thread
// }
taosRemoveRef(streamMetaId, pMeta->rid);
} }
void streamMetaCloseImpl(void* arg) { void streamMetaCloseImpl(void* arg) {
@ -525,10 +580,11 @@ void streamMetaCloseImpl(void* arg) {
streamMetaClear(pMeta); streamMetaClear(pMeta);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
tdbAbort(pMeta->db, pMeta->txn); // already log the error, ignore here
tdbTbClose(pMeta->pTaskDb); (void) tdbAbort(pMeta->db, pMeta->txn);
tdbTbClose(pMeta->pCheckpointDb); (void) tdbTbClose(pMeta->pTaskDb);
tdbClose(pMeta->db); (void) tdbTbClose(pMeta->pCheckpointDb);
(void) tdbClose(pMeta->db);
taosArrayDestroy(pMeta->pTaskList); taosArrayDestroy(pMeta->pTaskList);
taosArrayDestroy(pMeta->chkpSaved); taosArrayDestroy(pMeta->chkpSaved);
@ -552,7 +608,7 @@ void streamMetaCloseImpl(void* arg) {
bkdMgtDestroy(pMeta->bkdChkptMgt); bkdMgtDestroy(pMeta->bkdChkptMgt);
pMeta->role = NODE_ROLE_UNINIT; pMeta->role = NODE_ROLE_UNINIT;
taosThreadRwlockDestroy(&pMeta->lock); (void) taosThreadRwlockDestroy(&pMeta->lock);
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
stDebug("vgId:%d end to close stream meta", vgId); stDebug("vgId:%d end to close stream meta", vgId);
@ -568,9 +624,10 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
if (code < 0) { if (code < 0) {
return -1; return -1;
} }
buf = taosMemoryCalloc(1, len); buf = taosMemoryCalloc(1, len);
if (buf == NULL) { if (buf == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
@ -579,13 +636,19 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, len); tEncoderInit(&encoder, buf, len);
tEncodeStreamTask(&encoder, pTask); code = tEncodeStreamTask(&encoder, pTask);
tEncoderClear(&encoder); tEncoderClear(&encoder);
if (code == -1) {
stError("s-task:%s vgId:%d task meta encode failed, code:%s", pTask->id.idStr, vgId, tstrerror(code));
return TSDB_CODE_INVALID_MSG;
}
int64_t id[2] = {pTask->id.streamId, pTask->id.taskId}; int64_t id[2] = {pTask->id.streamId, pTask->id.taskId};
code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn); code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
code = terrno;
stError("s-task:%s vgId:%d task meta save to disk failed, code:%s", pTask->id.idStr, vgId, tstrerror(terrno)); stError("s-task:%s vgId:%d task meta save to disk failed, code:%s", pTask->id.idStr, vgId, tstrerror(terrno));
} else { } else {
stDebug("s-task:%s vgId:%d task meta save to disk", pTask->id.idStr, vgId); stDebug("s-task:%s vgId:%d task meta save to disk", pTask->id.idStr, vgId);
@ -612,33 +675,44 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) {
int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded) { int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded) {
*pAdded = false; *pAdded = false;
int32_t code = 0;
STaskId id = streamTaskGetTaskId(pTask); STaskId id = streamTaskGetTaskId(pTask);
void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (p != NULL) { if (p != NULL) {
return 0; stDebug("s-task:%" PRIx64 " already exist in meta, no need to register", id.taskId);
return code;
} }
if (pMeta->buildTaskFn(pMeta->ahandle, pTask, ver) < 0) { if ((code = pMeta->buildTaskFn(pMeta->ahandle, pTask, ver)) != 0) {
return -1; return code;
} }
taosArrayPush(pMeta->pTaskList, &pTask->id); p = taosArrayPush(pMeta->pTaskList, &pTask->id);
taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES); if (p == NULL) {
stError("s-task:0x%"PRIx64" failed to register task into meta-list, code: out of memory", id.taskId);
if (streamMetaSaveTask(pMeta, pTask) < 0) { return TSDB_CODE_OUT_OF_MEMORY;
return -1;
} }
if (streamMetaCommit(pMeta) < 0) { code = taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES);
return -1; if (code) {
stError("s-task:0x%"PRIx64" failed to register task into meta-list, code: out of memory", id.taskId);
return code;
}
if ((code = streamMetaSaveTask(pMeta, pTask)) != 0) {
return code;
}
if ((code = streamMetaCommit(pMeta)) != 0) {
return code;
} }
if (pTask->info.fillHistory == 0) { if (pTask->info.fillHistory == 0) {
atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); (void) atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1);
} }
*pAdded = true; *pAdded = true;
return 0; return code;
} }
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) {
@ -703,7 +777,7 @@ static void doRemoveIdFromList(SArray* pTaskList, int32_t num, SStreamTaskId* id
static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) {
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
streamTaskSendCheckpointSourceRsp(pTask); (void) streamTaskSendCheckpointSourceRsp(pTask);
} }
return 0; return 0;
} }
@ -726,7 +800,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
} }
// handle the dropping event // handle the dropping event
streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL); (void) streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL);
} else { } else {
stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId); stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -765,12 +839,12 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
pTask = *ppTask; pTask = *ppTask;
// it is an fill-history task, remove the related stream task's id that points to it // it is an fill-history task, remove the related stream task's id that points to it
if (pTask->info.fillHistory == 0) { if (pTask->info.fillHistory == 0) {
atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); (void) atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1);
} }
taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); (void) taosHashRemove(pMeta->pTasksMap, &id, sizeof(id));
doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id);
streamMetaRemoveTask(pMeta, &id); (void) streamMetaRemoveTask(pMeta, &id);
ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList));
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -778,7 +852,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
ASSERT(pTask->status.timerActive == 0); ASSERT(pTask->status.timerActive == 0);
if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) {
stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt);
taosTmrStop(pTask->schedInfo.pDelayTimer); (void) taosTmrStop(pTask->schedInfo.pDelayTimer);
pTask->info.delaySchedParam = 0; pTask->info.delaySchedParam = 0;
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} }
@ -823,9 +897,11 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) {
int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
int64_t checkpointId = 0; int64_t checkpointId = 0;
int32_t code = 0;
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) {
stError("failed to open stream meta file, the latest checkpointId is 0, vgId:%d", pMeta->vgId);
return checkpointId; return checkpointId;
} }
@ -835,7 +911,13 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
int32_t vLen = 0; int32_t vLen = 0;
SDecoder decoder; SDecoder decoder;
tdbTbcMoveToFirst(pCur); code = tdbTbcMoveToFirst(pCur);
if (code) {
(void) tdbTbcClose(pCur);
stError("failed to open stream meta file cursor, the latest checkpointId is 0, vgId:%d", pMeta->vgId);
return checkpointId;
}
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
if (pVal == NULL || vLen == 0) { if (pVal == NULL || vLen == 0) {
break; break;
@ -854,8 +936,8 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
tdbFree(pKey); tdbFree(pKey);
tdbFree(pVal); tdbFree(pVal);
tdbTbcClose(pCur);
(void)tdbTbcClose(pCur);
return checkpointId; return checkpointId;
} }
@ -867,23 +949,34 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
void* pVal = NULL; void* pVal = NULL;
int32_t vLen = 0; int32_t vLen = 0;
SDecoder decoder; SDecoder decoder;
int32_t vgId = 0;
int32_t code = 0;
SArray* pRecycleList = NULL;
if (pMeta == NULL) { if (pMeta == NULL) {
return; return;
} }
SArray* pRecycleList = taosArrayInit(4, sizeof(STaskId)); pRecycleList = taosArrayInit(4, sizeof(STaskId));
int32_t vgId = pMeta->vgId;
vgId = pMeta->vgId;
stInfo("vgId:%d load stream tasks from meta files", vgId); stInfo("vgId:%d load stream tasks from meta files", vgId);
int32_t code = tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL); code = tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("vgId:%d failed to open stream meta, code:%s, not load any stream tasks", vgId, tstrerror(terrno)); stError("vgId:%d failed to open stream meta, code:%s, not load any stream tasks", vgId, tstrerror(terrno));
taosArrayDestroy(pRecycleList); taosArrayDestroy(pRecycleList);
return; return;
} }
tdbTbcMoveToFirst(pCur); code = tdbTbcMoveToFirst(pCur);
if (code) {
stError("vgId:%d failed to open stream meta cursor, code:%s, not load any stream tasks", vgId, tstrerror(terrno));
taosArrayDestroy(pRecycleList);
(void) tdbTbcClose(pCur);
return;
}
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
if (pVal == NULL || vLen == 0) { if (pVal == NULL || vLen == 0) {
break; break;
@ -913,7 +1006,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
STaskId id = streamTaskGetTaskId(pTask); STaskId id = streamTaskGetTaskId(pTask);
taosArrayPush(pRecycleList, &id); (void) taosArrayPush(pRecycleList, &id);
int32_t total = taosArrayGetSize(pRecycleList); int32_t total = taosArrayGetSize(pRecycleList);
stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total); stDebug("s-task:0x%x is already dropped, add into recycle list, total:%d", taskId, total);
@ -934,7 +1027,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
continue; continue;
} }
taosArrayPush(pMeta->pTaskList, &pTask->id); (void) taosArrayPush(pMeta->pTaskList, &pTask->id);
} else { } else {
// todo this should replace the existed object put by replay creating stream task msg from mnode // todo this should replace the existed object put by replay creating stream task msg from mnode
stError("s-task:0x%x already added into table meta by replaying WAL, need check", pTask->id.taskId); stError("s-task:0x%x already added into table meta by replaying WAL, need check", pTask->id.taskId);
@ -944,17 +1037,17 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) { if (taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES) != 0) {
stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno)); stError("s-task:0x%x failed to put into hashTable, code:%s, continue", pTask->id.taskId, tstrerror(terrno));
taosArrayPop(pMeta->pTaskList); (void) taosArrayPop(pMeta->pTaskList);
tFreeStreamTask(pTask); tFreeStreamTask(pTask);
continue; continue;
} }
if (pTask->info.fillHistory == 0) { if (pTask->info.fillHistory == 0) {
atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); (void) atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1);
} }
if (streamTaskShouldPause(pTask)) { if (streamTaskShouldPause(pTask)) {
atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); (void) atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
} }
ASSERT(pTask->status.downstreamReady == 0); ASSERT(pTask->status.downstreamReady == 0);
@ -970,7 +1063,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) {
if (taosArrayGetSize(pRecycleList) > 0) { if (taosArrayGetSize(pRecycleList) > 0) {
for (int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pRecycleList); ++i) {
STaskId* pId = taosArrayGet(pRecycleList, i); STaskId* pId = taosArrayGet(pRecycleList, i);
streamMetaRemoveTask(pMeta, pId); (void) streamMetaRemoveTask(pMeta, pId);
} }
} }
@ -998,7 +1091,7 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) {
SStreamTask* pTask = *(SStreamTask**)pIter; SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->status.timerActive >= 1) { if (pTask->status.timerActive >= 1) {
stDebug("s-task:%s in timer, blocking tasks in vgId:%d restart, set closing again", pTask->id.idStr, pMeta->vgId); stDebug("s-task:%s in timer, blocking tasks in vgId:%d restart, set closing again", pTask->id.idStr, pMeta->vgId);
streamTaskStop(pTask); (void) streamTaskStop(pTask);
inTimer = true; inTimer = true;
} }
} }
@ -1031,7 +1124,7 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) {
SStreamTask* pTask = *(SStreamTask**)pIter; SStreamTask* pTask = *(SStreamTask**)pIter;
stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr); stDebug("vgId:%d s-task:%s set task closing flag", vgId, pTask->id.idStr);
streamTaskStop(pTask); (void) streamTaskStop(pTask);
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
@ -1050,7 +1143,16 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) {
void streamMetaStartHb(SStreamMeta* pMeta) { void streamMetaStartHb(SStreamMeta* pMeta) {
int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t));
metaRefMgtAdd(pMeta->vgId, pRid); if (pRid == NULL) {
stError("vgId:%d failed to prepare the metaHb to mnode, hbMsg will not started, code: out of memory", pMeta->vgId);
return;
}
int32_t code = metaRefMgtAdd(pMeta->vgId, pRid);
if (code) {
return;
}
*pRid = pMeta->rid; *pRid = pMeta->rid;
streamMetaHbToMnode(pRid, NULL); streamMetaHbToMnode(pRid, NULL);
} }
@ -1069,7 +1171,7 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo, int32_t vgId) {
void streamMetaRLock(SStreamMeta* pMeta) { void streamMetaRLock(SStreamMeta* pMeta) {
// stTrace("vgId:%d meta-rlock", pMeta->vgId); // stTrace("vgId:%d meta-rlock", pMeta->vgId);
taosThreadRwlockRdlock(&pMeta->lock); (void) taosThreadRwlockRdlock(&pMeta->lock);
} }
void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaRUnLock(SStreamMeta* pMeta) {
@ -1084,30 +1186,13 @@ void streamMetaRUnLock(SStreamMeta* pMeta) {
void streamMetaWLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) {
// stTrace("vgId:%d meta-wlock", pMeta->vgId); // stTrace("vgId:%d meta-wlock", pMeta->vgId);
taosThreadRwlockWrlock(&pMeta->lock); (void) taosThreadRwlockWrlock(&pMeta->lock);
// stTrace("vgId:%d meta-wlock completed", pMeta->vgId); // stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
} }
void streamMetaWUnLock(SStreamMeta* pMeta) { void streamMetaWUnLock(SStreamMeta* pMeta) {
// stTrace("vgId:%d meta-wunlock", pMeta->vgId); // stTrace("vgId:%d meta-wunlock", pMeta->vgId);
taosThreadRwlockUnlock(&pMeta->lock); (void) taosThreadRwlockUnlock(&pMeta->lock);
}
static void execHelper(struct SSchedMsg* pSchedMsg) {
__async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle;
int32_t code = execFn(pSchedMsg->thandle);
if (code != 0 && pSchedMsg->msg != NULL) {
*(int32_t*)pSchedMsg->msg = code;
}
}
int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code) {
SSchedMsg schedMsg = {0};
schedMsg.fp = execHelper;
schedMsg.ahandle = fn;
schedMsg.thandle = param;
schedMsg.msg = code;
return taosScheduleTask(pMeta->qHandle, &schedMsg);
} }
int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) { int32_t streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta, SArray** pList) {
@ -1195,10 +1280,10 @@ static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64
taosHashClear(pMeta->startInfo.pFailedTaskSet); taosHashClear(pMeta->startInfo.pFailedTaskSet);
pMeta->startInfo.startTs = now; pMeta->startInfo.startTs = now;
streamMetaResetTaskStatus(pMeta); int32_t code = streamMetaResetTaskStatus(pMeta);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return TSDB_CODE_SUCCESS; return code;
} }
// restore the checkpoint id by negotiating the latest consensus checkpoint id // restore the checkpoint id by negotiating the latest consensus checkpoint id
@ -1233,7 +1318,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); (void) streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId);
continue; continue;
} }
@ -1256,7 +1341,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask); code = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId);
streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); (void) streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId);
continue; continue;
} }
@ -1274,7 +1359,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
pTask->id.idStr); pTask->id.idStr);
streamLaunchFillHistoryTask(pTask); (void) streamLaunchFillHistoryTask(pTask); // todo: how about retry launch fill-history task?
} }
(void) streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true); (void) streamMetaAddTaskLaunchResult(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->checkTs, pInfo->readyTs, true);
@ -1340,7 +1425,7 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) {
continue; continue;
} }
streamTaskStop(pTask); (void) streamTaskStop(pTask);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} }
@ -1380,7 +1465,7 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask); code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId); stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId);
streamMetaAddFailedTask(pMeta, streamId, taskId); (void) streamMetaAddFailedTask(pMeta, streamId, taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
@ -1471,7 +1556,10 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet;
STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready};
taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); int32_t code = taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs));
if (code) {
}
int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta); int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);
int32_t numOfRecv = taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet); int32_t numOfRecv = taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet);
@ -1491,14 +1579,14 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
streamMetaResetStartInfo(pStartInfo, pMeta->vgId); streamMetaResetStartInfo(pStartInfo, pMeta->vgId);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
pStartInfo->completeFn(pMeta); code = pStartInfo->completeFn(pMeta);
} else { } else {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", pMeta->vgId, taskId, stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", pMeta->vgId, taskId,
ready, numOfRecv, numOfTotal); ready, numOfRecv, numOfTotal);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta) { int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta) {
@ -1572,19 +1660,26 @@ void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SSt
int64_t startTs) { int64_t startTs) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
int32_t code = 0;
// keep the already updated info // keep the already updated info
STaskUpdateEntry entry = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId, .transId = transId}; STaskUpdateEntry entry = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId, .transId = transId};
taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); code = taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0);
if (code != 0) {
stError("s-task:%s failed to put updateTask into update list", id);
}
int64_t el = taosGetTimestampMs() - startTs; int64_t el = taosGetTimestampMs() - startTs;
if (pHTask != NULL) { if (pHTask != NULL) {
STaskUpdateEntry hEntry = {.streamId = pHTask->id.streamId, .taskId = pHTask->id.taskId, .transId = transId}; STaskUpdateEntry hEntry = {.streamId = pHTask->id.streamId, .taskId = pHTask->id.taskId, .transId = transId};
taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0); code = taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0);
if (code != 0) {
stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64 stError("s-task:%s failed to put updateTask into update list", id);
" ms", } else {
id, vgId, transId, el); stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64
" ms",
id, vgId, transId, el);
}
} else { } else {
stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms",
id, vgId, transId, el); id, vgId, transId, el);

View File

@ -130,7 +130,6 @@ int32_t streamGetFileSize(char* path, char* name, int64_t* sz) {
int32_t ret = 0; int32_t ret = 0;
char* fullname = taosMemoryCalloc(1, strlen(path) + 32); char* fullname = taosMemoryCalloc(1, strlen(path) + 32);
sprintf(fullname, "%s%s%s", path, TD_DIRSEP, name); sprintf(fullname, "%s%s%s", path, TD_DIRSEP, name);
ret = taosStatFile(fullname, sz, NULL, NULL); ret = taosStatFile(fullname, sz, NULL, NULL);
@ -185,48 +184,89 @@ void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) {
} }
int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) { int32_t snapFileGenMeta(SBackendSnapFile2* pSnapFile) {
void* p = NULL;
SBackendFileItem item = {0}; SBackendFileItem item = {0};
item.ref = 1; item.ref = 1;
// current // current
item.name = pSnapFile->pCurrent; item.name = pSnapFile->pCurrent;
item.type = ROCKSDB_CURRENT_TYPE; item.type = ROCKSDB_CURRENT_TYPE;
streamGetFileSize(pSnapFile->path, item.name, &item.size); int32_t code = streamGetFileSize(pSnapFile->path, item.name, &item.size);
taosArrayPush(pSnapFile->pFileList, &item); if (code) {
stError("failed to get file size");
return code;
}
p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// mainfest // mainfest
item.name = pSnapFile->pMainfest; item.name = pSnapFile->pMainfest;
item.type = ROCKSDB_MAINFEST_TYPE; item.type = ROCKSDB_MAINFEST_TYPE;
streamGetFileSize(pSnapFile->path, item.name, &item.size); code = streamGetFileSize(pSnapFile->path, item.name, &item.size);
taosArrayPush(pSnapFile->pFileList, &item); if (code) {
return code;
}
p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// options // options
item.name = pSnapFile->pOptions; item.name = pSnapFile->pOptions;
item.type = ROCKSDB_OPTIONS_TYPE; item.type = ROCKSDB_OPTIONS_TYPE;
streamGetFileSize(pSnapFile->path, item.name, &item.size); code = streamGetFileSize(pSnapFile->path, item.name, &item.size);
taosArrayPush(pSnapFile->pFileList, &item); if (code) {
return code;
}
p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
// sst // sst
for (int32_t i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) { for (int32_t i = 0; i < taosArrayGetSize(pSnapFile->pSst); i++) {
char* sst = taosArrayGetP(pSnapFile->pSst, i); char* sst = taosArrayGetP(pSnapFile->pSst, i);
item.name = sst; item.name = sst;
item.type = ROCKSDB_SST_TYPE; item.type = ROCKSDB_SST_TYPE;
streamGetFileSize(pSnapFile->path, item.name, &item.size); code = streamGetFileSize(pSnapFile->path, item.name, &item.size);
taosArrayPush(pSnapFile->pFileList, &item); if (code) {
return code;
}
p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
// meta // meta
item.name = pSnapFile->pCheckpointMeta; item.name = pSnapFile->pCheckpointMeta;
item.type = ROCKSDB_CHECKPOINT_META_TYPE; item.type = ROCKSDB_CHECKPOINT_META_TYPE;
if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) { if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) {
taosArrayPush(pSnapFile->pFileList, &item); p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
item.name = pSnapFile->pCheckpointSelfcheck; item.name = pSnapFile->pCheckpointSelfcheck;
item.type = ROCKSDB_CHECKPOINT_SELFCHECK_TYPE; item.type = ROCKSDB_CHECKPOINT_SELFCHECK_TYPE;
if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) { if (streamGetFileSize(pSnapFile->path, item.name, &item.size) == 0) {
taosArrayPush(pSnapFile->pFileList, &item); p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} }
return 0; return 0;
} }
int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) { int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) {
int32_t code = 0; int32_t code = 0;
TdDirPtr pDir = taosOpenDir(pSnapFile->path); TdDirPtr pDir = taosOpenDir(pSnapFile->path);
@ -288,12 +328,18 @@ int32_t snapFileReadMeta(SBackendSnapFile2* pSnapFile) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
break; break;
} }
taosArrayPush(pSnapFile->pSst, &sst);
void* p = taosArrayPush(pSnapFile->pSst, &sst);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
} }
} }
taosCloseDir(&pDir);
return code; return taosCloseDir(&pDir);
} }
int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) { int32_t streamBackendSnapInitFile(char* metaPath, SStreamTaskSnap* pSnap, SBackendSnapFile2* pSnapFile) {
int32_t code = 0; int32_t code = 0;
int32_t nBytes = 0; int32_t nBytes = 0;
@ -359,13 +405,16 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) {
} }
taosArrayDestroy(pSnap->pFileList); taosArrayDestroy(pSnap->pFileList);
taosArrayDestroy(pSnap->pSst); taosArrayDestroy(pSnap->pSst);
taosCloseFile(&pSnap->fd); int32_t code = taosCloseFile(&pSnap->fd);
if (code) {
return; stError("failed to close snapshot fd");
}
} }
int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) { int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) {
// impl later
int32_t code = 0; int32_t code = 0;
SArray* pDbSnapSet = NULL;
SArray* pSnapInfoSet = taosArrayInit(4, sizeof(SStreamTaskSnap)); SArray* pSnapInfoSet = taosArrayInit(4, sizeof(SStreamTaskSnap));
if (pSnapInfoSet == NULL) { if (pSnapInfoSet == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -374,15 +423,13 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta
code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet); code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet);
if (code != 0) { if (code != 0) {
stError("failed to do task db snap info, reason:%s", tstrerror(code)); stError("failed to do task db snap info, reason:%s", tstrerror(code));
taosArrayDestroy(pSnapInfoSet); goto _err;
return code;
} }
SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2));
if (pDbSnapSet == NULL) { if (pDbSnapSet == NULL) {
taosArrayDestroy(pSnapInfoSet);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return code; goto _err;
} }
for (int32_t i = 0; i < taosArrayGetSize(pSnapInfoSet); i++) { for (int32_t i = 0; i < taosArrayGetSize(pSnapInfoSet); i++) {
@ -391,16 +438,24 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta
SBackendSnapFile2 snapFile = {0}; SBackendSnapFile2 snapFile = {0};
code = streamBackendSnapInitFile(path, pSnap, &snapFile); code = streamBackendSnapInitFile(path, pSnap, &snapFile);
ASSERT(code == 0); ASSERT(code == 0);
taosArrayPush(pDbSnapSet, &snapFile);
void* p = taosArrayPush(pDbSnapSet, &snapFile);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
} }
pHandle->pDbSnapSet = pDbSnapSet; pHandle->pDbSnapSet = pDbSnapSet;
pHandle->pSnapInfoSet = pSnapInfoSet; pHandle->pSnapInfoSet = pSnapInfoSet;
pHandle->currIdx = 0; pHandle->currIdx = 0;
pHandle->pMeta = pMeta; pHandle->pMeta = pMeta;
return 0;
return code;
_err: _err:
taosArrayDestroy(pSnapInfoSet);
taosArrayDestroy(pDbSnapSet);
streamSnapHandleDestroy(pHandle); streamSnapHandleDestroy(pHandle);
return code; return code;
} }
@ -414,7 +469,8 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
} }
taosArrayDestroy(handle->pDbSnapSet); taosArrayDestroy(handle->pDbSnapSet);
} }
streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
(void) streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
if (handle->pSnapInfoSet) { if (handle->pSnapInfoSet) {
for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) { for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) {
SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i); SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i);
@ -422,8 +478,8 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
} }
taosArrayDestroy(handle->pSnapInfoSet); taosArrayDestroy(handle->pSnapInfoSet);
} }
taosMemoryFree(handle->metaPath); taosMemoryFree(handle->metaPath);
return;
} }
int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* path, SStreamSnapReader** ppReader) { int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* path, SStreamSnapReader** ppReader) {
@ -506,14 +562,22 @@ _NEXT:
item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx); item->name, (int64_t)pSnapFile->offset, item->size, pSnapFile->currFileIdx);
pSnapFile->offset += nread; pSnapFile->offset += nread;
if (pSnapFile->offset >= item->size || nread < kBlockSize) { if (pSnapFile->offset >= item->size || nread < kBlockSize) {
taosCloseFile(&pSnapFile->fd); code = taosCloseFile(&pSnapFile->fd);
if (code) {
stError("failed to close snapshot fd");
}
pSnapFile->offset = 0; pSnapFile->offset = 0;
pSnapFile->currFileIdx += 1; pSnapFile->currFileIdx += 1;
} }
} else { } else {
stDebug("%s no data read, close file no.%d, move to next file, open and read", STREAM_STATE_TRANSFER, stDebug("%s no data read, close file no.%d, move to next file, open and read", STREAM_STATE_TRANSFER,
pSnapFile->currFileIdx); pSnapFile->currFileIdx);
taosCloseFile(&pSnapFile->fd); code = taosCloseFile(&pSnapFile->fd);
if (code) {
stError("failed to close snapshot fd");
}
pSnapFile->offset = 0; pSnapFile->offset = 0;
pSnapFile->currFileIdx += 1; pSnapFile->currFileIdx += 1;
@ -577,14 +641,22 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); pHandle->pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2));
if (pHandle->pDbSnapSet == NULL) { if (pHandle->pDbSnapSet == NULL) {
streamSnapWriterClose(pWriter, 0); int32_t c = streamSnapWriterClose(pWriter, 0); // not override the error code, and igore this error code
if (c) {
stError("failed close snaphost writer");
}
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return code; return code;
} }
SBackendSnapFile2 snapFile = {0}; SBackendSnapFile2 snapFile = {0};
if (taosArrayPush(pHandle->pDbSnapSet, &snapFile) == NULL) { if (taosArrayPush(pHandle->pDbSnapSet, &snapFile) == NULL) {
streamSnapWriterClose(pWriter, 0); int32_t c = streamSnapWriterClose(pWriter, 0);
if (c) {
stError("failed close snaphost writer");
}
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return code; return code;
} }
@ -614,46 +686,62 @@ int32_t streamSnapWriteImpl(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t
pHdr->name, tstrerror(code)); pHdr->name, tstrerror(code));
} }
} }
if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) { if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) {
int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset); int64_t bytes = taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset);
if (bytes != pHdr->size) { if (bytes != pHdr->size) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code));
return code; goto _err;
} else { } else {
stInfo("succ to write data %s", pItem->name); stInfo("succ to write data %s", pItem->name);
} }
pSnapFile->offset += bytes; pSnapFile->offset += bytes;
} else { } else {
taosCloseFile(&pSnapFile->fd); code = taosCloseFile(&pSnapFile->fd);
if (code) {
stError("failed to close snapshot fd");
}
pSnapFile->offset = 0; pSnapFile->offset = 0;
pSnapFile->currFileIdx += 1; pSnapFile->currFileIdx += 1;
SBackendFileItem item = {0}; SBackendFileItem item = {0};
item.name = taosStrdup(pHdr->name); item.name = taosStrdup(pHdr->name);
item.type = pHdr->type; item.type = pHdr->type;
if (item.name == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayPush(pSnapFile->pFileList, &item); void* p = taosArrayPush(pSnapFile->pFileList, &item);
if (p == NULL) { // can NOT goto _err here.
return TSDB_CODE_OUT_OF_MEMORY;
}
SBackendFileItem* pItem = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx); SBackendFileItem* pItem2 = taosArrayGet(pSnapFile->pFileList, pSnapFile->currFileIdx);
pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); pSnapFile->fd = streamOpenFile(pSnapFile->path, pItem2->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pSnapFile->fd == NULL) { if (pSnapFile->fd == NULL) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP, stError("%s failed to open file name:%s%s%s, reason:%s", STREAM_STATE_TRANSFER, pSnapFile->path, TD_DIRSEP,
pHdr->name, tstrerror(code)); pHdr->name, tstrerror(code));
return code;
} }
// open fd again, let's close fd during handle errors.
if (taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset) != pHdr->size) { if (taosPWriteFile(pSnapFile->fd, pHdr->data, pHdr->size, pSnapFile->offset) != pHdr->size) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code)); stError("%s failed to write snap, file name:%s, reason:%s", STREAM_STATE_TRANSFER, pHdr->name, tstrerror(code));
return code; goto _err;
} }
stInfo("succ to write data %s", pItem->name);
stInfo("succ to write data %s", pItem2->name);
pSnapFile->offset += pHdr->size; pSnapFile->offset += pHdr->size;
} }
code = 0;
_EXIT: return TSDB_CODE_SUCCESS;
_err:
(void) taosCloseFile(&pSnapFile->fd);
return code; return code;
} }
@ -688,7 +776,10 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
item.name = taosStrdup((char*)ROCKSDB_CURRENT); item.name = taosStrdup((char*)ROCKSDB_CURRENT);
item.type = ROCKSDB_CURRENT_TYPE; item.type = ROCKSDB_CURRENT_TYPE;
taosArrayPush(pDbSnapFile->pFileList, &item); void* p = taosArrayPush(pDbSnapFile->pFileList, &item);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pDbSnapFile->inited = 1; pDbSnapFile->inited = 1;
return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile);
@ -697,9 +788,12 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile); return streamSnapWriteImpl(pWriter, pData, nData, pDbSnapFile);
} else { } else {
SBackendSnapFile2 snapFile = {0}; SBackendSnapFile2 snapFile = {0};
taosArrayPush(pHandle->pDbSnapSet, &snapFile); void* p = taosArrayPush(pHandle->pDbSnapSet, &snapFile);
pHandle->currIdx += 1; if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pHandle->currIdx += 1;
return streamSnapWrite(pWriter, pData, nData); return streamSnapWrite(pWriter, pData, nData);
} }
} }

View File

@ -29,19 +29,19 @@ typedef struct SLaunchHTaskInfo {
STaskId hTaskId; STaskId hTaskId;
} SLaunchHTaskInfo; } SLaunchHTaskInfo;
static int32_t streamSetParamForScanHistory(SStreamTask* pTask); static int32_t streamSetParamForScanHistory(SStreamTask* pTask);
static void streamTaskSetRangeStreamCalc(SStreamTask* pTask); static int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask);
static int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); static void initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated);
static SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, static int32_t createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId,
int32_t hTaskId); SLaunchHTaskInfo** pInfo);
static void tryLaunchHistoryTask(void* param, void* tmrId); static void tryLaunchHistoryTask(void* param, void* tmrId);
static void doExecScanhistoryInFuture(void* param, void* tmrId); static void doExecScanhistoryInFuture(void* param, void* tmrId);
static int32_t doStartScanHistoryTask(SStreamTask* pTask); static int32_t doStartScanHistoryTask(SStreamTask* pTask);
static int32_t streamTaskStartScanHistory(SStreamTask* pTask); static int32_t streamTaskStartScanHistory(SStreamTask* pTask);
static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask); static void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask);
static int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask); static int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask);
static void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); static void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now);
static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now); static void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now);
static int32_t streamTaskSetReady(SStreamTask* pTask) { static int32_t streamTaskSetReady(SStreamTask* pTask) {
int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask);
@ -65,22 +65,19 @@ static int32_t streamTaskSetReady(SStreamTask* pTask) {
int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) {
SStreamScanHistoryReq req; SStreamScanHistoryReq req;
int32_t code = 0;
initScanHistoryReq(pTask, &req, igUntreated); initScanHistoryReq(pTask, &req, igUntreated);
int32_t len = sizeof(SStreamScanHistoryReq); int32_t len = sizeof(SStreamScanHistoryReq);
void* serializedReq = rpcMallocCont(len); void* serializedReq = rpcMallocCont(len);
if (serializedReq == NULL) { if (serializedReq == NULL) {
return -1; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(serializedReq, &req, len); memcpy(serializedReq, &req, len);
SRpcMsg rpcMsg = {.contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_SCAN_HISTORY}; SRpcMsg rpcMsg = {.contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_SCAN_HISTORY};
if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { return tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg);
/*ASSERT(0);*/
}
return 0;
} }
int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) { int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) {
@ -109,8 +106,8 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration)
pTask->schedHistoryInfo.pTimer = pTask->schedHistoryInfo.pTimer =
taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer);
} else { } else {
taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer,
&pTask->schedHistoryInfo.pTimer); &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr");
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -135,9 +132,19 @@ int32_t streamTaskStartScanHistory(SStreamTask* pTask) {
int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t code = 0;
streamTaskSetReady(pTask); code = streamTaskSetReady(pTask);
streamTaskSetRangeStreamCalc(pTask); if (code) {
stError("s-task:%s failed to set task status ready", id);
return code;
}
code = streamTaskSetRangeStreamCalc(pTask);
if (code) {
stError("s-task:%s failed to set the time range for stream task", id);
return code;
}
SStreamTaskState p = streamTaskGetStatus(pTask); SStreamTaskState p = streamTaskGetStatus(pTask);
ASSERT(p.state == TASK_STATUS__READY); ASSERT(p.state == TASK_STATUS__READY);
@ -155,19 +162,23 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) {
stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p.name, schedStatus); stDebug("s-task:%s level:%d status:%s sched-status:%d", id, pTask->info.taskLevel, p.name, schedStatus);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
// set the state to be ready // set the state to be ready
streamTaskSetReady(pTask); int32_t code = streamTaskSetReady(pTask);
streamTaskSetRangeStreamCalc(pTask); if (code == 0) {
code = streamTaskSetRangeStreamCalc(pTask);
}
SStreamTaskState p = streamTaskGetStatus(pTask); if (code == 0) {
ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1)); SStreamTaskState p = streamTaskGetStatus(pTask);
ASSERT((p.state == TASK_STATUS__SCAN_HISTORY) && (pTask->info.fillHistory == 1));
stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name); stDebug("s-task:%s fill-history task enters into scan-history data stage, status:%s", pTask->id.idStr, p.name);
streamTaskStartScanHistory(pTask); code = streamTaskStartScanHistory(pTask);
}
// NOTE: there will be an deadlock if launch fill history here. // NOTE: there will be an deadlock if launch fill history here.
// start the related fill-history task, when current task is ready // start the related fill-history task, when current task is ready
@ -175,7 +186,7 @@ int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) {
// streamLaunchFillHistoryTask(pTask); // streamLaunchFillHistoryTask(pTask);
// } // }
return TSDB_CODE_SUCCESS; return code;
} }
// common // common
@ -212,8 +223,7 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId,
pStatus.name); pStatus.name);
(void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); return streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return -1; // todo set the correct error code
} }
stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", idStr, hStreamId, hTaskId); stDebug("s-task:%s start to launch related fill-history task:0x%" PRIx64 "-0x%x", idStr, hStreamId, hTaskId);
@ -257,12 +267,11 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) {
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated) { void initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated) {
pReq->msgHead.vgId = pTask->info.nodeId; pReq->msgHead.vgId = pTask->info.nodeId;
pReq->streamId = pTask->id.streamId; pReq->streamId = pTask->id.streamId;
pReq->taskId = pTask->id.taskId; pReq->taskId = pTask->id.taskId;
pReq->igUntreated = igUntreated; pReq->igUntreated = igUntreated;
return 0;
} }
void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) { void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) {
@ -281,7 +290,10 @@ void checkFillhistoryTaskStatus(SStreamTask* pTask, SStreamTask* pHTask) {
} }
// check if downstream tasks have been ready // check if downstream tasks have been ready
streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST); int32_t code = streamTaskHandleEvent(pHTask->status.pSM, TASK_EVENT_INIT_SCANHIST);
if (code) {
stError("s-task:%s handle event init_scanhist failed", pTask->id.idStr);
}
} }
void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now) { void notRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, int64_t now) {
@ -316,7 +328,8 @@ void doRetryLaunchFillHistoryTask(SStreamTask* pTask, SLaunchHTaskInfo* pInfo, i
stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d",
pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes);
taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer,
pTask->pMeta->vgId, " start-history-task-tmr");
} }
} }
@ -367,7 +380,8 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
pHTaskInfo->tickCount -= 1; pHTaskInfo->tickCount -= 1;
if (pHTaskInfo->tickCount > 0) { if (pHTaskInfo->tickCount > 0) {
taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer); streamTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamTimer, &pHTaskInfo->pTimer,
pTask->pMeta->vgId, " start-history-task-tmr");
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return; return;
} }
@ -417,21 +431,21 @@ void tryLaunchHistoryTask(void* param, void* tmrId) {
taosMemoryFree(pInfo); taosMemoryFree(pInfo);
} }
SLaunchHTaskInfo* createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId) { int32_t createHTaskLaunchInfo(SStreamMeta* pMeta, STaskId* pTaskId, int64_t hStreamId, int32_t hTaskId,
SLaunchHTaskInfo* pInfo = taosMemoryCalloc(1, sizeof(SLaunchHTaskInfo)); SLaunchHTaskInfo** pInfo) {
if (pInfo == NULL) { *pInfo = taosMemoryCalloc(1, sizeof(SLaunchHTaskInfo));
terrno = TSDB_CODE_OUT_OF_MEMORY; if ((*pInfo) == NULL) {
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pInfo->id.streamId = pTaskId->streamId; (*pInfo)->id.streamId = pTaskId->streamId;
pInfo->id.taskId = pTaskId->taskId; (*pInfo)->id.taskId = pTaskId->taskId;
pInfo->hTaskId.streamId = hStreamId; (*pInfo)->hTaskId.streamId = hStreamId;
pInfo->hTaskId.taskId = hTaskId; (*pInfo)->hTaskId.taskId = hTaskId;
pInfo->pMeta = pMeta; (*pInfo)->pMeta = pMeta;
return pInfo; return TSDB_CODE_SUCCESS;
} }
int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
@ -440,16 +454,18 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
const char* idStr = pTask->id.idStr; const char* idStr = pTask->id.idStr;
int64_t hStreamId = pTask->hTaskInfo.id.streamId; int64_t hStreamId = pTask->hTaskInfo.id.streamId;
int32_t hTaskId = pTask->hTaskInfo.id.taskId; int32_t hTaskId = pTask->hTaskInfo.id.taskId;
SLaunchHTaskInfo* pInfo = NULL;
ASSERT(hTaskId != 0); ASSERT(hTaskId != 0);
stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId); stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId);
STaskId id = streamTaskGetTaskId(pTask); STaskId id = streamTaskGetTaskId(pTask);
SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); int32_t code = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId, &pInfo);
if (pInfo == NULL) { if (code) {
stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr);
(void) streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false); (void)streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, false);
return terrno; return code;
} }
// set the launch time info // set the launch time info
@ -475,7 +491,8 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) {
} else { // timer exists } else { // timer exists
ASSERT(pTask->status.timerActive >= 1); ASSERT(pTask->status.timerActive >= 1);
stDebug("s-task:%s set timer active flag, task timer not null", idStr); stDebug("s-task:%s set timer active flag, task timer not null", idStr);
taosTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer); streamTmrReset(tryLaunchHistoryTask, WAIT_FOR_MINIMAL_INTERVAL, pInfo, streamTimer, &pTask->hTaskInfo.pTimer,
pTask->pMeta->vgId, " start-history-task-tmr");
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -510,7 +527,7 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVe
} }
} }
void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { int32_t streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
SDataRange* pRange = &pTask->dataRange; SDataRange* pRange = &pTask->dataRange;
if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) { if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) {
@ -523,10 +540,12 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
"window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 "-%" PRId64, "window:%" PRId64 "-%" PRId64 ", verRange:%" PRId64 "-%" PRId64,
pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer); pTask->id.idStr, pRange->window.skey, pRange->window.ekey, pRange->range.minVer, pRange->range.maxVer);
} }
return TSDB_CODE_SUCCESS;
} else { } else {
ASSERT(pTask->info.fillHistory == 0); ASSERT(pTask->info.fillHistory == 0);
if (pTask->info.taskLevel >= TASK_LEVEL__AGG) { if (pTask->info.taskLevel >= TASK_LEVEL__AGG) {
return; return TSDB_CODE_SUCCESS;
} }
stDebug("s-task:%s level:%d related fill-history task exists, stream task timeWindow:%" PRId64 " - %" PRId64 stDebug("s-task:%s level:%d related fill-history task exists, stream task timeWindow:%" PRId64 " - %" PRId64
@ -536,7 +555,7 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) {
SVersionRange verRange = pRange->range; SVersionRange verRange = pRange->range;
STimeWindow win = pRange->window; STimeWindow win = pRange->window;
streamSetParamForStreamScannerStep2(pTask, &verRange, &win); return streamSetParamForStreamScannerStep2(pTask, &verRange, &win);
} }
} }
@ -554,7 +573,10 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) {
} }
if (pTask->schedHistoryInfo.numOfTicks <= 0) { if (pTask->schedHistoryInfo.numOfTicks <= 0) {
streamStartScanHistoryAsync(pTask, 0); int32_t code = streamStartScanHistoryAsync(pTask, 0);
if (code) {
stError("s-task:%s async start history task failed", pTask->id.idStr);
}
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stDebug("s-task:%s fill-history:%d start scan-history data, out of tmr, ref:%d", pTask->id.idStr, stDebug("s-task:%s fill-history:%d start scan-history data, out of tmr, ref:%d", pTask->id.idStr,
@ -563,18 +585,26 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) {
// release the task. // release the task.
streamMetaReleaseTask(pTask->pMeta, pTask); streamMetaReleaseTask(pTask->pMeta, pTask);
} else { } else {
taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, streamTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer,
&pTask->schedHistoryInfo.pTimer); &pTask->schedHistoryInfo.pTimer, pTask->pMeta->vgId, " start-history-task-tmr");
} }
} }
int32_t doStartScanHistoryTask(SStreamTask* pTask) { int32_t doStartScanHistoryTask(SStreamTask* pTask) {
int32_t code = 0;
SVersionRange* pRange = &pTask->dataRange.range; SVersionRange* pRange = &pTask->dataRange.range;
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
streamSetParamForScanHistory(pTask); code = streamSetParamForScanHistory(pTask);
if (code) {
return code;
}
} }
streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window); code = streamSetParamForStreamScannerStep1(pTask, pRange, &pTask->dataRange.window);
int32_t code = streamStartScanHistoryAsync(pTask, 0); if (code) {
return code; return code;
}
return streamStartScanHistoryAsync(pTask, 0);
} }

View File

@ -29,20 +29,20 @@ static void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo);
static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) { static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) {
int32_t childId = taosArrayGetSize(pArray); int32_t childId = taosArrayGetSize(pArray);
pTask->info.selfChildId = childId; pTask->info.selfChildId = childId;
taosArrayPush(pArray, &pTask); void* p = taosArrayPush(pArray, &pTask);
return 0; return (p == NULL)? TSDB_CODE_OUT_OF_MEMORY:TSDB_CODE_SUCCESS;
} }
static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEpSet, bool* pUpdated) { static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEpSet, bool* pUpdated) {
char buf[512] = {0}; char buf[512] = {0};
if (pTask->info.nodeId == nodeId) { // execution task should be moved away if (pTask->info.nodeId == nodeId) { // execution task should be moved away
bool isEqual = isEpsetEqual(&pTask->info.epSet, pEpSet); bool isEqual = isEpsetEqual(&pTask->info.epSet, pEpSet);
epsetToStr(pEpSet, buf, tListLen(buf)); (void)epsetToStr(pEpSet, buf, tListLen(buf));
if (!isEqual) { if (!isEqual) {
(*pUpdated) = true; (*pUpdated) = true;
char tmp[512] = {0}; char tmp[512] = {0};
epsetToStr(&pTask->info.epSet, tmp, tListLen(tmp)); (void) epsetToStr(&pTask->info.epSet, tmp, tListLen(tmp)); // only for log file, ignore errors
epsetAssign(&pTask->info.epSet, pEpSet); epsetAssign(&pTask->info.epSet, pEpSet);
stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s, old:%s", pTask->id.taskId, nodeId, buf, tmp); stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s, old:%s", pTask->id.taskId, nodeId, buf, tmp);
@ -127,7 +127,10 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool
pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
pTask->taskCheckInfo.pList = taosArrayInit(4, sizeof(SDownstreamStatusInfo)); pTask->taskCheckInfo.pList = taosArrayInit(4, sizeof(SDownstreamStatusInfo));
taosThreadMutexInit(&pTask->taskCheckInfo.checkInfoLock, NULL); code = taosThreadMutexInit(&pTask->taskCheckInfo.checkInfoLock, NULL);
if (code) {
return code;
}
if (fillHistory) { if (fillHistory) {
ASSERT(hasFillhistory); ASSERT(hasFillhistory);
@ -135,7 +138,7 @@ int32_t tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool
epsetAssign(&(pTask->info.mnodeEpset), pEpset); epsetAssign(&(pTask->info.mnodeEpset), pEpset);
addToTaskset(pTaskList, pTask); code = addToTaskset(pTaskList, pTask);
*p = pTask; *p = pTask;
return code; return code;
@ -221,17 +224,17 @@ void tFreeStreamTask(SStreamTask* pTask) {
} }
if (pTask->schedInfo.pDelayTimer != NULL) { if (pTask->schedInfo.pDelayTimer != NULL) {
taosTmrStop(pTask->schedInfo.pDelayTimer); (void) taosTmrStop(pTask->schedInfo.pDelayTimer);
pTask->schedInfo.pDelayTimer = NULL; pTask->schedInfo.pDelayTimer = NULL;
} }
if (pTask->hTaskInfo.pTimer != NULL) { if (pTask->hTaskInfo.pTimer != NULL) {
/*bool ret = */ taosTmrStop(pTask->hTaskInfo.pTimer); (void) taosTmrStop(pTask->hTaskInfo.pTimer);
pTask->hTaskInfo.pTimer = NULL; pTask->hTaskInfo.pTimer = NULL;
} }
if (pTask->msgInfo.pRetryTmr != NULL) { if (pTask->msgInfo.pRetryTmr != NULL) {
/*bool ret = */ taosTmrStop(pTask->msgInfo.pRetryTmr); (void) taosTmrStop(pTask->msgInfo.pRetryTmr);
pTask->msgInfo.pRetryTmr = NULL; pTask->msgInfo.pRetryTmr = NULL;
} }
@ -394,10 +397,12 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
return terrno; return terrno;
} }
taosThreadMutexInit(&pTask->msgInfo.lock, NULL); code = taosThreadMutexInit(&pTask->msgInfo.lock, NULL);
if (code) {
return code;
}
TdThreadMutexAttr attr = {0}; TdThreadMutexAttr attr = {0};
code = taosThreadMutexAttrInit(&attr); code = taosThreadMutexAttrInit(&attr);
if (code != 0) { if (code != 0) {
stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code)); stError("s-task:%s initElapsed mutex attr failed, code:%s", pTask->id.idStr, tstrerror(code));
@ -410,8 +415,16 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
return code; return code;
} }
taosThreadMutexInit(&pTask->lock, &attr); code = taosThreadMutexInit(&pTask->lock, &attr);
taosThreadMutexAttrDestroy(&attr); if (code) {
return code;
}
code = taosThreadMutexAttrDestroy(&attr);
if (code) {
return code;
}
streamTaskOpenAllUpstreamInput(pTask); streamTaskOpenAllUpstreamInput(pTask);
STaskOutputInfo* pOutputInfo = &pTask->outputInfo; STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
@ -424,7 +437,11 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
// 2MiB per second for sink task // 2MiB per second for sink task
// 50 times sink operator per second // 50 times sink operator per second
streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr); code = streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr);
if (code) {
return code;
}
pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset)); pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset));
if (pOutputInfo->pNodeEpsetUpdateList == NULL) { if (pOutputInfo->pNodeEpsetUpdateList == NULL) {
stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr, stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr,
@ -474,13 +491,13 @@ int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstre
pTask->upstreamInfo.pList = taosArrayInit(4, POINTER_BYTES); pTask->upstreamInfo.pList = taosArrayInit(4, POINTER_BYTES);
} }
taosArrayPush(pTask->upstreamInfo.pList, &pEpInfo); void* p = taosArrayPush(pTask->upstreamInfo.pList, &pEpInfo);
return TSDB_CODE_SUCCESS; return (p == NULL)? TSDB_CODE_OUT_OF_MEMORY:TSDB_CODE_SUCCESS;
} }
void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) { void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) {
char buf[512] = {0}; char buf[512] = {0};
epsetToStr(pEpSet, buf, tListLen(buf)); (void) epsetToStr(pEpSet, buf, tListLen(buf)); // ignore error since it is only for log file.
int32_t numOfUpstream = taosArrayGetSize(pTask->upstreamInfo.pList); int32_t numOfUpstream = taosArrayGetSize(pTask->upstreamInfo.pList);
for (int32_t i = 0; i < numOfUpstream; ++i) { for (int32_t i = 0; i < numOfUpstream; ++i) {
@ -491,7 +508,7 @@ void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpS
*pUpdated = true; *pUpdated = true;
char tmp[512] = {0}; char tmp[512] = {0};
epsetToStr(&pInfo->epSet, tmp, tListLen(tmp)); (void) epsetToStr(&pInfo->epSet, tmp, tListLen(tmp));
epsetAssign(&pInfo->epSet, pEpSet); epsetAssign(&pInfo->epSet, pEpSet);
stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s old:%s", pTask->id.taskId, stDebug("s-task:0x%x update the upstreamInfo taskId:0x%x(nodeId:%d) newEpset:%s old:%s", pTask->id.taskId,
@ -526,7 +543,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo
void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) { void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) {
char buf[512] = {0}; char buf[512] = {0};
epsetToStr(pEpSet, buf, tListLen(buf)); (void) epsetToStr(pEpSet, buf, tListLen(buf)); // ignore the error since only for log files.
int32_t id = pTask->id.taskId; int32_t id = pTask->id.taskId;
int8_t type = pTask->outputInfo.type; int8_t type = pTask->outputInfo.type;
@ -542,7 +559,7 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
if (!isEqual) { if (!isEqual) {
*pUpdated = true; *pUpdated = true;
char tmp[512] = {0}; char tmp[512] = {0};
epsetToStr(&pVgInfo->epSet, tmp, tListLen(tmp)); (void) epsetToStr(&pVgInfo->epSet, tmp, tListLen(tmp));
epsetAssign(&pVgInfo->epSet, pEpSet); epsetAssign(&pVgInfo->epSet, pEpSet);
stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pVgInfo->taskId, stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pVgInfo->taskId,
@ -562,7 +579,7 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
*pUpdated = true; *pUpdated = true;
char tmp[512] = {0}; char tmp[512] = {0};
epsetToStr(&pDispatcher->epSet, tmp, tListLen(tmp)); (void) epsetToStr(&pDispatcher->epSet, tmp, tListLen(tmp));
epsetAssign(&pDispatcher->epSet, pEpSet); epsetAssign(&pDispatcher->epSet, pEpSet);
stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pDispatcher->taskId, stDebug("s-task:0x%x update dispatch info, task:0x%x(nodeId:%d) newEpset:%s old:%s", id, pDispatcher->taskId,
@ -580,8 +597,16 @@ int32_t streamTaskStop(SStreamTask* pTask) {
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP); int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_STOP);
qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS); if (code) {
stError("failed to handle STOP event, s-task:%s", id);
}
code = qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
if (code) {
stError("s-task:%s failed to kill task related query handle", id);
}
while (!streamTaskIsIdle(pTask)) { while (!streamTaskIsIdle(pTask)) {
stDebug("s-task:%s level:%d wait for task to be idle and then close, check again in 100ms", id, stDebug("s-task:%s level:%d wait for task to be idle and then close, check again in 100ms", id,
pTask->info.taskLevel); pTask->info.taskLevel);
@ -590,7 +615,7 @@ int32_t streamTaskStop(SStreamTask* pTask) {
int64_t el = taosGetTimestampMs() - st; int64_t el = taosGetTimestampMs() - st;
stDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms", vgId, id, el); stDebug("vgId:%d s-task:%s is closed in %" PRId64 " ms", vgId, id, el);
return 0; return code;
} }
bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) { bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) {
@ -607,7 +632,10 @@ bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList) {
bool updated = false; bool updated = false;
for (int32_t i = 0; i < taosArrayGetSize(pNodeList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pNodeList); ++i) {
SNodeUpdateInfo* pInfo = taosArrayGet(pNodeList, i); SNodeUpdateInfo* pInfo = taosArrayGet(pNodeList, i);
doUpdateTaskEpset(pTask, pInfo->nodeId, &pInfo->newEp, &updated); int32_t code = doUpdateTaskEpset(pTask, pInfo->nodeId, &pInfo->newEp, &updated);
if (code) {
stError("s-task:0x%x failed to update the task nodeEp epset, code:%s", pTask->id.taskId, tstrerror(code));
}
} }
return updated; return updated;
@ -704,10 +732,11 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
} }
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) { int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
int32_t code = 0;
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId}; STaskId sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
if (pTask->info.fillHistory == 0) { if (pTask->info.fillHistory == 0) {
return TSDB_CODE_SUCCESS; return code;
} }
SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId)); SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId));
@ -725,11 +754,11 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
(*ppStreamTask)->status.taskStatus = TASK_STATUS__READY; (*ppStreamTask)->status.taskStatus = TASK_STATUS__READY;
} }
streamMetaSaveTask(pMeta, *ppStreamTask); code = streamMetaSaveTask(pMeta, *ppStreamTask);
streamMutexUnlock(&(*ppStreamTask)->lock); streamMutexUnlock(&(*ppStreamTask)->lock);
} }
return TSDB_CODE_SUCCESS; return code;
} }
int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, int64_t resetRelHalt) { int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, int64_t resetRelHalt) {
@ -797,8 +826,7 @@ int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpoin
initRpcMsg(&msg, TDMT_MND_STREAM_CHKPT_REPORT, buf, tlen); initRpcMsg(&msg, TDMT_MND_STREAM_CHKPT_REPORT, buf, tlen);
stDebug("s-task:%s vgId:%d build and send task checkpoint-report to mnode", id, vgId); stDebug("s-task:%s vgId:%d build and send task checkpoint-report to mnode", id, vgId);
tmsgSendReq(&pTask->info.mnodeEpset, &msg); return tmsgSendReq(&pTask->info.mnodeEpset, &msg);
return 0;
} }
STaskId streamTaskGetTaskId(const SStreamTask* pTask) { STaskId streamTaskGetTaskId(const SStreamTask* pTask) {
@ -880,6 +908,7 @@ STaskStatusEntry streamTaskGetStatusEntry(SStreamTask* pTask) {
static int32_t taskPauseCallback(SStreamTask* pTask, void* param) { static int32_t taskPauseCallback(SStreamTask* pTask, void* param) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
int32_t code = 0;
int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1); int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
stInfo("vgId:%d s-task:%s pause stream task. paused task num:%d", pMeta->vgId, pTask->id.idStr, num); stInfo("vgId:%d s-task:%s pause stream task. paused task num:%d", pMeta->vgId, pTask->id.idStr, num);
@ -887,15 +916,15 @@ static int32_t taskPauseCallback(SStreamTask* pTask, void* param) {
// in case of fill-history task, stop the tsdb file scan operation. // in case of fill-history task, stop the tsdb file scan operation.
if (pTask->info.fillHistory == 1) { if (pTask->info.fillHistory == 1) {
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
qKillTask(pExecutor, TSDB_CODE_SUCCESS); code = qKillTask(pExecutor, TSDB_CODE_SUCCESS);
} }
stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr); stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr);
return TSDB_CODE_SUCCESS; return code;
} }
void streamTaskPause(SStreamTask* pTask) { void streamTaskPause(SStreamTask* pTask) {
streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL); (void) streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL);
} }
void streamTaskResume(SStreamTask* pTask) { void streamTaskResume(SStreamTask* pTask) {
@ -949,8 +978,7 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) {
initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CHKPT, buf, tlen); initRpcMsg(&msg, TDMT_MND_STREAM_REQ_CHKPT, buf, tlen);
stDebug("s-task:%s vgId:%d build and send task checkpoint req", id, vgId); stDebug("s-task:%s vgId:%d build and send task checkpoint req", id, vgId);
tmsgSendReq(&pTask->info.mnodeEpset, &msg); return tmsgSendReq(&pTask->info.mnodeEpset, &msg);
return 0;
} }
void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) { void streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId, SStreamUpstreamEpInfo** pEpInfo) {
@ -1044,7 +1072,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { pTask->status.removeBackendFiles = true; } void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { pTask->status.removeBackendFiles = true; }
int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) { void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) {
if (pTransId != NULL) { if (pTransId != NULL) {
*pTransId = pTask->chkInfo.pActiveInfo->transId; *pTransId = pTask->chkInfo.pActiveInfo->transId;
} }
@ -1052,8 +1080,6 @@ int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTr
if (pCheckpointId != NULL) { if (pCheckpointId != NULL) {
*pCheckpointId = pTask->chkInfo.pActiveInfo->activeId; *pCheckpointId = pTask->chkInfo.pActiveInfo->activeId;
} }
return TSDB_CODE_SUCCESS;
} }
int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId) { int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId) {
@ -1084,7 +1110,7 @@ int32_t streamTaskCreateActiveChkptInfo(SActiveCheckpointInfo** pRes) {
pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo)); pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo));
*pRes = pInfo; *pRes = pInfo;
return TSDB_CODE_SUCCESS; return code;
} }
void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
@ -1101,12 +1127,12 @@ void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) {
pInfo->pCheckpointReadyRecvList = NULL; pInfo->pCheckpointReadyRecvList = NULL;
if (pInfo->pChkptTriggerTmr != NULL) { if (pInfo->pChkptTriggerTmr != NULL) {
taosTmrStop(pInfo->pChkptTriggerTmr); (void) taosTmrStop(pInfo->pChkptTriggerTmr);
pInfo->pChkptTriggerTmr = NULL; pInfo->pChkptTriggerTmr = NULL;
} }
if (pInfo->pSendReadyMsgTmr != NULL) { if (pInfo->pSendReadyMsgTmr != NULL) {
taosTmrStop(pInfo->pSendReadyMsgTmr); (void) taosTmrStop(pInfo->pSendReadyMsgTmr);
pInfo->pSendReadyMsgTmr = NULL; pInfo->pSendReadyMsgTmr = NULL;
} }

View File

@ -35,8 +35,9 @@ void streamTimerCleanUp() {
streamTimer = NULL; streamTimer = NULL;
} }
tmr_h streamTimerGetInstance() { int32_t streamTimerGetInstance(tmr_h* pTmr) {
return streamTimer; *pTmr = streamTimer;
return TSDB_CODE_SUCCESS;
} }
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId, void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,

View File

@ -36,7 +36,6 @@ static int64_t adjustExpEntries(int64_t entries) { return TMIN(DEFAULT_EXPECTED_
int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) { int compareKeyTs(void* pTs1, void* pTs2, void* pPkVal, __compar_fn_t cmpPkFn) {
return compareInt64Val(pTs1, pTs2); return compareInt64Val(pTs1, pTs2);
;
} }
int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) { int compareKeyTsAndPk(void* pValue1, void* pTs, void* pPkVal, __compar_fn_t cmpPkFn) {

View File

@ -33,7 +33,7 @@ class TDTestCase:
self.testcasePath = os.path.split(__file__)[0] self.testcasePath = os.path.split(__file__)[0]
self.testcaseFilename = os.path.split(__file__)[-1] self.testcaseFilename = os.path.split(__file__)[-1]
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.db = "sel_null" self.db = "sel_null"
def insert_data(self,database,vgroups): def insert_data(self,database,vgroups):
@ -46,10 +46,10 @@ class TDTestCase:
for i in range(5): for i in range(5):
tdSql.execute('''create table %s.stb0_%d using %s.stb0 tags(%d,'varchar%d',%d,%d, %d, %d,%d,'binary%d','nchar%d',%d,%d,%d ) ;'''%(database,i,database,i,i,i,i,i,i,i,i,i,i,i,i)) tdSql.execute('''create table %s.stb0_%d using %s.stb0 tags(%d,'varchar%d',%d,%d, %d, %d,%d,'binary%d','nchar%d',%d,%d,%d ) ;'''%(database,i,database,i,i,i,i,i,i,i,i,i,i,i,i))
# insert data # insert data
for i in range(num_random): for i in range(num_random):
for j in range(50): for j in range(50):
tdSql.execute('''insert into %s.stb0_0 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) tdSql.execute('''insert into %s.stb0_0 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j))
tdSql.execute('''insert into %s.stb0_1 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) tdSql.execute('''insert into %s.stb0_1 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j))
tdSql.execute('''insert into %s.stb0_2 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j)) tdSql.execute('''insert into %s.stb0_2 (ts , c1 , c0) values(now, %d, %d) ;''' % (database,j,j))
@ -60,378 +60,378 @@ class TDTestCase:
tdSql.checkData(0,0,5*num_random*50) tdSql.checkData(0,0,5*num_random*50)
tdSql.query("select count(*) from %s.stb0_0;"%database) tdSql.query("select count(*) from %s.stb0_0;"%database)
tdSql.checkData(0,0,num_random*50) tdSql.checkData(0,0,num_random*50)
def ts_3085(self,database): def ts_3085(self,database):
sql = "select count(c0null) from(select * from %s.stb0 limit 20,4) "%(database) sql = "select count(c0null) from(select * from %s.stb0 limit 20,4) "%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
offset = random.randint(10,100) offset = random.randint(10,100)
for i in range(offset): for i in range(offset):
sql = "select count(c0null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) sql = "select count(c0null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select count(c1null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) sql = "select count(c1null) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select count(c0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) sql = "select count(c0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,i) tdSql.checkData(0,0,i)
sql = "select count(c1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) sql = "select count(c1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,i) tdSql.checkData(0,0,i)
sql = "select count(t0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) sql = "select count(t0) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,i) tdSql.checkData(0,0,i)
sql = "select count(t1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i) sql = "select count(t1) from(select * from %s.stb0 limit %d,%d) "%(database,offset,i)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,i) tdSql.checkData(0,0,i)
def ts_2974_max(self,database): def ts_2974_max(self,database):
sql = "select max(c0) from %s.stb0 where ts<now;"%(database) sql = "select max(c0) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,49) tdSql.checkData(0,0,49)
sql = "select max(c0),ts from %s.stb0 where ts<now;"%(database) sql = "select max(c0),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,49) tdSql.checkData(0,0,49)
sql = "select max(c1) from %s.stb0 where ts<now;"%(database) sql = "select max(c1) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,49) tdSql.checkData(0,0,49)
sql = "select max(c1),ts from %s.stb0 where ts<now;"%(database) sql = "select max(c1),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,49) tdSql.checkData(0,0,49)
sql = "select max(c0null) from %s.stb0 where ts<now;"%(database) sql = "select max(c0null) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select max(c0null),ts from %s.stb0 where ts<now;"%(database) sql = "select max(c0null),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select max(c1null) from %s.stb0 where ts<now;"%(database) sql = "select max(c1null) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select max(c1null),ts from %s.stb0 where ts<now;"%(database) sql = "select max(c1null),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select max(t0) from %s.stb0 where ts<now;"%(database) sql = "select max(t0) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t0),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t0),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t1) from %s.stb0 where ts<now;"%(database) sql = "select max(t1) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t1),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t1),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_bool) from %s.stb0 where ts<now;"%(database) sql = "select max(t_bool) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_bool),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_bool),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_binary) from %s.stb0 where ts<now;"%(database) sql = "select max(t_binary) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_binary),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_binary),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_nchar) from %s.stb0 where ts<now;"%(database) sql = "select max(t_nchar) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_nchar),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_nchar),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select max(t_int) from %s.stb0 where ts<now;"%(database) sql = "select max(t_int) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_int),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_int),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_int) from %s.stb0 where ts<now;"%(database) sql = "select max(t_int) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_int),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_int),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_int) from %s.stb0 where ts<now;"%(database) sql = "select max(t_int) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_int),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_int),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_bigint) from %s.stb0 where ts<now;"%(database) sql = "select max(t_bigint) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_bigint),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_bigint),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_smallint) from %s.stb0 where ts<now;"%(database) sql = "select max(t_smallint) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_smallint),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_smallint),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_tinyint) from %s.stb0 where ts<now;"%(database) sql = "select max(t_tinyint) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_tinyint),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_tinyint),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_float) from %s.stb0 where ts<now;"%(database) sql = "select max(t_float) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_float),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_float),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_double) from %s.stb0 where ts<now;"%(database) sql = "select max(t_double) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
sql = "select max(t_double),ts from %s.stb0 where ts<now;"%(database) sql = "select max(t_double),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,4) tdSql.checkData(0,0,4)
def ts_2974_min(self,database): def ts_2974_min(self,database):
sql = "select min(c0) from %s.stb0 where ts<now;"%(database) sql = "select min(c0) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(c0),ts from %s.stb0 where ts<now;"%(database) sql = "select min(c0),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(c1) from %s.stb0 where ts<now;"%(database) sql = "select min(c1) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(c1),ts from %s.stb0 where ts<now;"%(database) sql = "select min(c1),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(c0null) from %s.stb0 where ts<now;"%(database) sql = "select min(c0null) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select min(c0null),ts from %s.stb0 where ts<now;"%(database) sql = "select min(c0null),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select min(c1null) from %s.stb0 where ts<now;"%(database) sql = "select min(c1null) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select min(c1null),ts from %s.stb0 where ts<now;"%(database) sql = "select min(c1null),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,"None") tdSql.checkData(0,0,"None")
sql = "select min(t0) from %s.stb0 where ts<now;"%(database) sql = "select min(t0) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t0),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t0),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t1) from %s.stb0 where ts<now;"%(database) sql = "select min(t1) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t1),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t1),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_bool) from %s.stb0 where ts<now;"%(database) sql = "select min(t_bool) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_bool),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_bool),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_binary) from %s.stb0 where ts<now;"%(database) sql = "select min(t_binary) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_binary),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_binary),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_nchar) from %s.stb0 where ts<now;"%(database) sql = "select min(t_nchar) from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_nchar),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_nchar),ts from %s.stb0 where ts<now;"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select min(t_int) from %s.stb0 where ts<now;"%(database) sql = "select min(t_int) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_int),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_int),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_int) from %s.stb0 where ts<now;"%(database) sql = "select min(t_int) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_int),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_int),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_int) from %s.stb0 where ts<now;"%(database) sql = "select min(t_int) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_int),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_int),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_bigint) from %s.stb0 where ts<now;"%(database) sql = "select min(t_bigint) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_bigint),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_bigint),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_smallint) from %s.stb0 where ts<now;"%(database) sql = "select min(t_smallint) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_smallint),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_smallint),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_tinyint) from %s.stb0 where ts<now;"%(database) sql = "select min(t_tinyint) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_tinyint),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_tinyint),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_float) from %s.stb0 where ts<now;"%(database) sql = "select min(t_float) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_float),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_float),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_double) from %s.stb0 where ts<now;"%(database) sql = "select min(t_double) from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
sql = "select min(t_double),ts from %s.stb0 where ts<now;"%(database) sql = "select min(t_double),ts from %s.stb0 where ts<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,0) tdSql.checkData(0,0,0)
def ts_2601(self,database): def ts_2601(self,database):
tdSql.query("alter local 'keepcolumnname' '0';") tdSql.query("alter local 'keepcolumnname' '0';")
sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now);"%(database) sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now);"%(database)
tdSql.error(sql) tdSql.error(sql)
sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now order by ts );"%(database) sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now order by ts );"%(database)
tdSql.error(sql) tdSql.error(sql)
tdSql.query("alter local 'keepcolumnname' '1';") tdSql.query("alter local 'keepcolumnname' '1';")
sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now);"%(database) sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now);"%(database)
tdSql.query(sql) tdSql.query(sql)
sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now order by ts );"%(database) sql = "select ts,c0 from (select last(*) from %s.stb0 where ts<now order by ts );"%(database)
tdSql.query(sql) tdSql.query(sql)
def ts_3108(self,database): def ts_3108(self,database):
sql = "select count(*) from %s.stb0 where to_unixtimestamp('2023-01-01 00:00:00.000')<now;"%(database) sql = "select count(*) from %s.stb0 where to_unixtimestamp('2023-01-01 00:00:00.000')<now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,2500) tdSql.checkData(0,0,2500)
sql = "select count(*) from %s.stb0 where to_unixtimestamp('2023-01-01 00:00:00.000')>now;"%(database) sql = "select count(*) from %s.stb0 where to_unixtimestamp('2023-01-01 00:00:00.000')>now;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkRows(0) tdSql.checkRows(0)
sql = "select count(*) from %s.stb0 where to_unixtimestamp('2024-01-01 00:00:00.000')<now+1y;"%(database) sql = "select count(*) from %s.stb0 where to_unixtimestamp('2024-01-01 00:00:00.000')<now+1y;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkData(0,0,2500) tdSql.checkData(0,0,2500)
sql = "select count(*) from %s.stb0 where to_unixtimestamp('2024-01-01 00:00:00.000')>now+1y;"%(database) sql = "select count(*) from %s.stb0 where to_unixtimestamp('2024-01-01 00:00:00.000')>now+1y;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkRows(0) tdSql.checkRows(0)
def ts_3110(self,database): def ts_3110(self,database):
sql1 = "select * from %s.stb0 order by ts desc limit 2;"%(database) sql1 = "select * from %s.stb0 order by ts desc limit 2;"%(database)
tdSql.query(sql1) tdSql.query(sql1)
data1_0_0 = tdSql.getData(0,0) data1_0_0 = tdSql.getData(0,0)
data1_1_0 = tdSql.getData(1,0) data1_1_0 = tdSql.getData(1,0)
sql2 = "select * from (select * from %s.stb0 order by ts desc limit 2) order by ts;"%(database) sql2 = "select * from (select * from %s.stb0 order by ts desc limit 2) order by ts;"%(database)
tdSql.query(sql2) tdSql.query(sql2)
data2_0_0 = tdSql.getData(0,0) data2_0_0 = tdSql.getData(0,0)
data2_1_0 = tdSql.getData(1,0) data2_1_0 = tdSql.getData(1,0)
if (data1_0_0 == data2_1_0) and (data1_1_0 == data2_0_0): if (data1_0_0 == data2_1_0) and (data1_1_0 == data2_0_0):
tdLog.info("ts_3110: success") tdLog.info("ts_3110: success")
else: else:
tdLog.exit("ts_3110: sql1 result:'%s' not equal sql2 result:'%s'" % (sql1,sql2)) tdLog.exit("ts_3110: sql1 result:'%s' not equal sql2 result:'%s'" % (sql1,sql2))
def ts_3036(self,database): def ts_3036(self,database):
sql1 = "select ts , c0 , c1 , c0null , c1null from (select ts , c0 , c1 , c0null , c1null from %s.stb0_0 where ts between now -1d and now +1d \ sql1 = "select ts , c0 , c1 , c0null , c1null from (select ts , c0 , c1 , c0null , c1null from %s.stb0_0 where ts between now -1d and now +1d \
union all select ts , c0 , c1 , c0null , c1null from %s.stb0_1 where ts between now -1d and now +1d \ union all select ts , c0 , c1 , c0null , c1null from %s.stb0_1 where ts between now -1d and now +1d \
union all select ts , c0 , c1 , c0null , c1null from %s.stb0_2 where ts between now -1d and now +1d ) tt \ union all select ts , c0 , c1 , c0null , c1null from %s.stb0_2 where ts between now -1d and now +1d ) tt \
where ts < now order by tt.ts desc limit 2;"%(database,database,database) where ts < now order by tt.ts desc limit 2;"%(database,database,database)
tdSql.query(sql1) tdSql.query(sql1)
data1_0_0 = tdSql.getData(0,0) data1_0_0 = tdSql.getData(0,0)
data1_1_0 = tdSql.getData(1,0) data1_1_0 = tdSql.getData(1,0)
sql2 = "select ts , c0 , c1 , c0null , c1null from (select tbname as tb, ts , c0 , c1 , c0null , c1null from %s.stb0 where ts > now \ sql2 = "select ts , c0 , c1 , c0null , c1null from (select tbname as tb, ts , c0 , c1 , c0null , c1null from %s.stb0 where ts > now \
union all select tbname as tb, ts , c0 , c1 , c0null , c1null from %s.stb0 where ts = now \ union all select tbname as tb, ts , c0 , c1 , c0null , c1null from %s.stb0 where ts = now \
union all select tbname as tb, ts , c0 , c1 , c0null , c1null from %s.stb0 where ts < now ) tt \ union all select tbname as tb, ts , c0 , c1 , c0null , c1null from %s.stb0 where ts < now ) tt \
where tt.ts between now -1d and now +1d and tt.tb in ('stb0_0','stb0_1','stb0_2') order by tt.ts desc limit 2;"%(database,database,database) where tt.ts between now -1d and now +1d and tt.tb in ('stb0_0','stb0_1','stb0_2') order by tt.ts desc limit 2;"%(database,database,database)
tdSql.query(sql2) tdSql.query(sql2)
data2_0_0 = tdSql.getData(0,0) data2_0_0 = tdSql.getData(0,0)
data2_1_0 = tdSql.getData(1,0) data2_1_0 = tdSql.getData(1,0)
sql3 = "select ts , c0 , c1 , c0null , c1null from %s.stb0 \ sql3 = "select ts , c0 , c1 , c0null , c1null from %s.stb0 \
where ts between now -1d and now +1d and tbname in ('stb0_0','stb0_1','stb0_2') order by ts desc limit 2;"%(database) where ts between now -1d and now +1d and tbname in ('stb0_0','stb0_1','stb0_2') order by ts desc limit 2;"%(database)
tdSql.query(sql3) tdSql.query(sql3)
data3_0_0 = tdSql.getData(0,0) data3_0_0 = tdSql.getData(0,0)
data3_1_0 = tdSql.getData(1,0) data3_1_0 = tdSql.getData(1,0)
if (data1_0_0 == data2_0_0 == data3_0_0) and (data1_1_0 == data2_1_0 == data3_1_0): if (data1_0_0 == data2_0_0 == data3_0_0) and (data1_1_0 == data2_1_0 == data3_1_0):
tdLog.info("ts_3036: success") tdLog.info("ts_3036: success")
else: else:
tdLog.exit("ts_3036: sql1 result:'%s' not equal sql2 result:'%s' or not equal sql3 result:'%s'" % (sql1,sql2,sql3)) tdLog.exit("ts_3036: sql1 result:'%s' not equal sql2 result:'%s' or not equal sql3 result:'%s'" % (sql1,sql2,sql3))
def ts_23569(self,database): def ts_23569(self,database):
tdSql.query("alter local 'keepcolumnname' '0';") tdSql.query("alter local 'keepcolumnname' '0';")
sql = "alter table %s.stb0 drop tag t10;"%(database) sql = "alter table %s.stb0 drop tag t10;"%(database)
tdSql.error(sql) tdSql.error(sql)
error_msg = tdSql.error(sql) error_msg = tdSql.error(sql)
include_msg = 'Invalid tag name' include_msg = 'Invalid tag name'
if include_msg in error_msg: if include_msg in error_msg:
tdLog.info("ts_23569: success") tdLog.info("ts_23569: success")
else: else:
tdLog.exit("ts_23569: include_msg:'%s' not in error_msg:'%s'" % (include_msg,error_msg)) tdLog.exit("ts_23569: include_msg:'%s' not in error_msg:'%s'" % (include_msg,error_msg))
tdSql.query("alter local 'keepcolumnname' '1';") tdSql.query("alter local 'keepcolumnname' '1';")
sql = "alter table %s.stb0 drop tag t10;"%(database) sql = "alter table %s.stb0 drop tag t10;"%(database)
tdSql.error(sql) tdSql.error(sql)
error_msg = tdSql.error(sql) error_msg = tdSql.error(sql)
include_msg = 'Invalid tag name' include_msg = 'Invalid tag name'
if include_msg in error_msg: if include_msg in error_msg:
tdLog.info("ts_23569: success") tdLog.info("ts_23569: success")
else: else:
tdLog.exit("ts_23569: include_msg:'%s' not in error_msg:'%s'" % (include_msg,error_msg)) tdLog.exit("ts_23569: include_msg:'%s' not in error_msg:'%s'" % (include_msg,error_msg))
def ts_23505(self,database): def ts_23505(self,database):
sql = "create table %s.`12345` (`567` timestamp,num int);"%(database) sql = "create table %s.`12345` (`567` timestamp,num int);"%(database)
tdSql.execute(sql) tdSql.execute(sql)
sql = "insert into %s.12345 values (now,1);"%(database)
tdSql.error(sql)
sql = "insert into %s.`12345` values (now,1);"%(database)
tdSql.execute(sql)
sql = "select * from %s.`12345` order by `567` desc limit 2;"%(database)
tdSql.query(sql)
tdSql.checkData(0,1,1)
sql = "drop table %s.`12345` ;"%(database)
tdSql.execute(sql)
sql = "select * from %s.`12345` order by `567` desc limit 2;"%(database)
tdSql.error(sql)
def td_27939(self,database): sql = "insert into %s.12345 values (now,1);"%(database)
tdSql.error(sql)
sql = "insert into %s.`12345` values (now,1);"%(database)
tdSql.execute(sql)
sql = "select * from %s.`12345` order by `567` desc limit 2;"%(database)
tdSql.query(sql)
tdSql.checkData(0,1,1)
sql = "drop table %s.`12345` ;"%(database)
tdSql.execute(sql)
sql = "select * from %s.`12345` order by `567` desc limit 2;"%(database)
tdSql.error(sql)
def td_27939(self,database):
sql = "create table %s.`test1eq2` (`ts` timestamp, id int);"%(database) sql = "create table %s.`test1eq2` (`ts` timestamp, id int);"%(database)
tdSql.execute(sql) tdSql.execute(sql)
sql = "insert into %s.test1eq2 values (now,1);"%(database) sql = "insert into %s.test1eq2 values (now,1);"%(database)
tdSql.execute(sql) tdSql.execute(sql)
sql = "insert into %s.`test1eq2` values (now,2);"%(database) sql = "insert into %s.`test1eq2` values (now,2);"%(database)
tdSql.execute(sql) tdSql.execute(sql)
sql = "select * from %s.`test1eq2` where 1=2;"%(database) sql = "select * from %s.`test1eq2` where 1=2;"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkRows(0) tdSql.checkRows(0)
sql = "select * from (select * from %s.`test1eq2` where 1=2);"%(database) sql = "select * from (select * from %s.`test1eq2` where 1=2);"%(database)
tdSql.query(sql) tdSql.query(sql)
tdSql.checkRows(0) tdSql.checkRows(0)
sql = "drop table %s.`test1eq2` ;"%(database) sql = "drop table %s.`test1eq2` ;"%(database)
tdSql.execute(sql) tdSql.execute(sql)
def run(self): def run(self):
startTime = time.time() startTime = time.time()
os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
self.insert_data("%s" %self.db,2) self.insert_data("%s" %self.db,2)
self.ts_3085("%s" %self.db) self.ts_3085("%s" %self.db)
self.ts_2974_max("%s" %self.db) self.ts_2974_max("%s" %self.db)
self.ts_2974_min("%s" %self.db) self.ts_2974_min("%s" %self.db)
@ -443,9 +443,9 @@ class TDTestCase:
self.ts_3036("%s" %self.db) self.ts_3036("%s" %self.db)
self.td_27939("%s" %self.db) self.td_27939("%s" %self.db)
tdSql.query("flush database %s" %self.db) tdSql.query("flush database %s" %self.db)
self.ts_2974_max("%s" %self.db) self.ts_2974_max("%s" %self.db)
self.ts_2974_min("%s" %self.db) self.ts_2974_min("%s" %self.db)
self.ts_3085("%s" %self.db) self.ts_3085("%s" %self.db)
@ -461,7 +461,7 @@ class TDTestCase:
self.test_select_as_chinese_characters(); self.test_select_as_chinese_characters();
endTime = time.time() endTime = time.time()
print("total time %ds" % (endTime - startTime)) print("total time %ds" % (endTime - startTime))
def test_select_as_chinese_characters(self): def test_select_as_chinese_characters(self):
tdSql.execute("use sel_null") tdSql.execute("use sel_null")
tdSql.query("select ts as 时间戳, c0 as c第一列, t0 标签1 from sel_null.stb0_0 limit 10", queryTimes=1) tdSql.query("select ts as 时间戳, c0 as c第一列, t0 标签1 from sel_null.stb0_0 limit 10", queryTimes=1)