xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:[TD-31017]process return value in client for tmq

This commit is contained in:
wangmm0220 2024-07-25 00:21:01 +08:00
parent b873356d80 16d98d53a9
commit b3d1cd9ed1
240 changed files with 14099 additions and 9082 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tdatablock.h
View File

@ -49,7 +49,7 @@ typedef struct SBlockOrderInfo {
#define colDataSetNull_f_s(c_, r_) \
do { \
colDataSetNull_f((c_)->nullbitmap, r_); \
memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \
(void)memset(((char*)(c_)->pData) + (c_)->info.bytes * (r_), 0, (c_)->info.bytes); \
} while (0)
#define colDataClearNull_f(bm_, r_) \

15
include/common/tmsg.h
View File

@ -2828,19 +2828,24 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
return tlen;
}
static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
static FORCE_INLINE int32_t tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->cgroup);
buf = taosDecodeStringTo(buf, pReq->clientId);
int32_t topicNum;
int32_t topicNum = 0;
buf = taosDecodeFixedI32(buf, &topicNum);
pReq->topicNames = taosArrayInit(topicNum, sizeof(void*));
if (pReq->topicNames == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < topicNum; i++) {
char* name;
char* name = NULL;
buf = taosDecodeString(buf, &name);
taosArrayPush(pReq->topicNames, &name);
if (taosArrayPush(pReq->topicNames, &name) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
}
buf = taosDecodeFixedI8(buf, &pReq->withTbName);
@ -2849,7 +2854,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq
buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg);
buf = taosDecodeFixedI8(buf, &pReq->enableReplay);
buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta);
return buf;
return 0;
}
typedef struct {

4
include/common/ttime.h
View File

@ -86,11 +86,11 @@ void deltaToUtcInitOnce();
char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit);
int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);
struct STm {
struct tm tm;

5
include/dnode/qnode/qnode.h
View File

@ -34,9 +34,10 @@ typedef struct {
* @brief Start one Qnode in Dnode.
*
* @param pOption Option of the qnode.
* @return SQnode* The qnode object.
* @param pQnode The qnode object.
* @return int32_t The error code.
*/
SQnode *qndOpen(const SQnodeOpt *pOption);
int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode);
/**
* @brief Stop Qnode in Dnode.

2
include/libs/executor/storageapi.h
View File

@ -195,7 +195,7 @@ typedef struct SStoreCacheReader {
int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray *pFuncTypeList, SColumnInfo* pPkCol, int32_t numOfPks);
void *(*closeReader)(void *pReader);
void (*closeReader)(void *pReader);
int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUidList);
int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables);

2
include/libs/function/function.h
View File

@ -36,7 +36,7 @@ typedef struct SFuncExecEnv {
} SFuncExecEnv;
typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
typedef int32_t (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

6
include/libs/function/functionMgt.h
View File

@ -138,6 +138,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_CACHE_LAST_ROW,
FUNCTION_TYPE_CACHE_LAST,
FUNCTION_TYPE_TABLE_COUNT,
FUNCTION_TYPE_GROUP_CONST_VALUE,
// distributed splitting functions
FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000,
@ -256,9 +257,10 @@ bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId);
bool fmIsProcessByRowFunc(int32_t funcId);
bool fmisSelectGroupConstValueFunc(int32_t funcId);
void getLastCacheDataType(SDataType* pType, int32_t pkBytes);
SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList);
int32_t createFunction(const char* pName, SNodeList* pParameterList, SFunctionNode** pFunc);
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc);
@ -271,7 +273,7 @@ typedef enum EFuncDataRequired {
} EFuncDataRequired;
EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
EFuncDataRequired fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo);
int32_t fmFuncDynDataRequired(int32_t funcId, void* pRes, SDataBlockInfo* pBlockInfo, int32_t *reqStatus);
int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);

2
include/libs/function/tudf.h
View File

@ -77,7 +77,7 @@ void freeUdfInterBuf(SUdfInterBuf *buf);
// high level APIs
bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
int32_t udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock);

7
include/libs/stream/tstream.h
View File

@ -686,7 +686,7 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
// checkpoint related
int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId);
void streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId);
int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId);
int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId);
bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId);
@ -770,7 +770,7 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta);
int32_t streamTaskSendRestoreChkptMsg(SStreamTask* pTask);
// timer
tmr_h streamTimerGetInstance();
int32_t streamTimerGetInstance(tmr_h* pTmr);
void streamTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId, int32_t vgId,
const char* pMsg);
@ -809,6 +809,9 @@ void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp);
int32_t streamProcessHeartbeatRsp(SStreamMeta* pMeta, SMStreamHbRspMsg* pRsp);
int32_t streamTaskSendCheckpointsourceRsp(SStreamTask* pTask);
void streamMutexLock(TdThreadMutex *pMutex);
void streamMutexUnlock(TdThreadMutex *pMutex);
void streamMutexDestroy(TdThreadMutex *pMutex);
#ifdef __cplusplus
}

39
include/libs/wal/wal.h
View File

@ -12,6 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_WAL_H_
#define _TD_WAL_H_
@ -20,19 +21,11 @@
#include "tdef.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
// clang-format off
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
// clang-format on
#define WAL_PROTO_VER 0
#define WAL_NOSUFFIX_LEN 20
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
@ -131,7 +124,6 @@ typedef struct SWal {
typedef struct {
int64_t refId;
int64_t refVer;
// int64_t refFile;
SWal *pWal;
} SWalRef;
@ -143,10 +135,8 @@ typedef struct {
int8_t enableRef;
} SWalFilterCond;
typedef struct SWalReader SWalReader;
// todo hide this struct
struct SWalReader {
typedef struct SWalReader {
SWal *pWal;
int64_t readerId;
TdFilePtr pLogFile;
@ -159,7 +149,7 @@ struct SWalReader {
TdThreadMutex mutex;
SWalFilterCond cond;
SWalCkHead *pHead;
};
} SWalReader;
// module initialization
int32_t walInit();
@ -172,17 +162,9 @@ int32_t walPersist(SWal *);
void walClose(SWal *);
// write interfaces
// By assigning index by the caller, wal gurantees linearizability
int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body,
int32_t bodyLen);
// Assign version automatically and return to caller,
// -1 will be returned for failed writes
int64_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
void walFsync(SWal *, bool force);
int32_t walAppendLog(SWal *, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen);
int32_t walFsync(SWal *, bool force);
// apis for lifecycle management
int32_t walCommit(SWal *, int64_t ver);
@ -191,17 +173,13 @@ int32_t walRollback(SWal *, int64_t ver);
int32_t walBeginSnapshot(SWal *, int64_t ver, int64_t logRetention);
int32_t walEndSnapshot(SWal *);
int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
// for tq
int32_t walApplyVer(SWal *, int64_t ver);
// int32_t walDataCorrupted(SWal*);
// wal reader
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id);
void walCloseReader(SWalReader *pRead);
void walReadReset(SWalReader *pReader);
int32_t walReadVer(SWalReader *pRead, int64_t ver);
void decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const char *func);
int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver);
int32_t walNextValidMsg(SWalReader *pRead);
int64_t walReaderGetCurrentVer(const SWalReader *pReader);
@ -216,12 +194,11 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver);
int32_t walFetchBody(SWalReader *pRead);
int32_t walSkipFetchBody(SWalReader *pRead);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
SWalRef *walOpenRef(SWal *);
void walCloseRef(SWal *pWal, int64_t refId);
int32_t walSetRefVer(SWalRef *, int64_t ver);
void walRefFirstVer(SWal *, SWalRef *);
void walRefLastVer(SWal *, SWalRef *);
// helper function for raft
bool walLogExist(SWal *, int64_t ver);

1
include/os/os.h
View File

@ -97,6 +97,7 @@ extern "C" {
#include <nmmintrin.h>
#endif
#include "osThread.h"
#include "osAtomic.h"

4
include/os/osEnv.h
View File

@ -54,7 +54,7 @@ extern SDiskSpace tsDataSpace;
extern SDiskSpace tsLogSpace;
extern SDiskSpace tsTempSpace;
void osDefaultInit();
int32_t osDefaultInit();
void osUpdate();
void osCleanup();
@ -66,7 +66,7 @@ bool osLogSpaceSufficient();
bool osDataSpaceSufficient();
bool osTempSpaceSufficient();
void osSetTimezone(const char *timezone);
int32_t osSetTimezone(const char *timezone);
void osSetSystemLocale(const char *inLocale, const char *inCharSet);
void osSetProcPath(int32_t argc, char **argv);

1
include/os/osString.h
View File

@ -61,6 +61,7 @@ typedef enum { M2C = 0, C2M } ConvType;
#define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src))
#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size))
#define TAOS_STRCAT(_dst, _src) ((void)strcat(_dst, _src))
char *tstrdup(const char *src);
int32_t taosUcs4len(TdUcs4 *ucs4);

8
include/os/osTime.h
View File

@ -65,28 +65,28 @@ int32_t taosGetTimestampSec();
//@return timestamp in millisecond
static FORCE_INLINE int64_t taosGetTimestampMs() {
struct timeval systemTime;
taosGetTimeOfDay(&systemTime);
(void)taosGetTimeOfDay(&systemTime);
return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec / 1000;
}
//@return timestamp in microsecond
static FORCE_INLINE int64_t taosGetTimestampUs() {
struct timeval systemTime;
taosGetTimeOfDay(&systemTime);
(void)taosGetTimeOfDay(&systemTime);
return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec;
}
//@return timestamp in nanosecond
static FORCE_INLINE int64_t taosGetTimestampNs() {
struct timespec systemTime = {0};
taosClockGetTime(CLOCK_REALTIME, &systemTime);
(void)taosClockGetTime(CLOCK_REALTIME, &systemTime);
return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec;
}
//@return timestamp of monotonic clock in millisecond
static FORCE_INLINE int64_t taosGetMonoTimestampMs() {
struct timespec systemTime = {0};
taosClockGetTime(CLOCK_MONOTONIC, &systemTime);
(void)taosClockGetTime(CLOCK_MONOTONIC, &systemTime);
return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_nsec / 1000000;
}

2
include/os/osTimezone.h
View File

@ -55,7 +55,7 @@ enum TdTimezone {
};
void taosGetSystemTimezone(char *outTimezone, enum TdTimezone *tsTimezone);
void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone);
int32_t taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight, enum TdTimezone *tsTimezone);
#ifdef __cplusplus
}

2
include/util/tRealloc.h
View File

@ -41,7 +41,7 @@ static FORCE_INLINE int32_t tRealloc(uint8_t **ppBuf, int64_t size) {
pBuf = (uint8_t *)taosMemoryRealloc(pBuf, bsize + sizeof(int64_t));
if (pBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}

5
include/util/taoserror.h
View File

@ -860,6 +860,11 @@ int32_t taosGetErrSize();
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)
#define TSDB_CODE_FUNC_TO_CHAR_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2809)
#define TSDB_CODE_FUNC_TIME_UNIT_INVALID TAOS_DEF_ERROR_CODE(0, 0x280A)
#define TSDB_CODE_FUNC_TIME_UNIT_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x280B)
#define TSDB_CODE_FUNC_INVALID_VALUE_RANGE TAOS_DEF_ERROR_CODE(0, 0x280C)
#define TSDB_CODE_FUNC_SETUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x280D)
//udf
#define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901)

2
include/util/tarray2.h
View File

@ -55,7 +55,7 @@ static FORCE_INLINE int32_t tarray2_make_room(void *arr, int32_t expSize, int32_
capacity <<= 1;
}
void *p = taosMemoryRealloc(a->data, capacity * eleSize);
if (p == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (p == NULL) return terrno;
a->capacity = capacity;
a->data = p;
return 0;

3
include/util/tdef.h
View File

@ -556,7 +556,8 @@ typedef struct {
char name[TSDB_LOG_VAR_LEN];
} SLogVar;
#define TMQ_SEPARATOR ':'
#define TMQ_SEPARATOR ":"
#define TMQ_SEPARATOR_CHAR ':'
enum {
SND_WORKER_TYPE__SHARED = 1,

6
include/util/tdigest.h
View File

@ -64,10 +64,10 @@ typedef struct TDigest {
} TDigest;
TDigest *tdigestNewFrom(void *pBuf, int32_t compression);
void tdigestAdd(TDigest *t, double x, int64_t w);
void tdigestMerge(TDigest *t1, TDigest *t2);
int32_t tdigestAdd(TDigest *t, double x, int64_t w);
int32_t tdigestMerge(TDigest *t1, TDigest *t2);
double tdigestQuantile(TDigest *t, double q);
void tdigestCompress(TDigest *t);
int32_t tdigestCompress(TDigest *t);
void tdigestFreeFrom(TDigest *t);
void tdigestAutoFill(TDigest *t, int32_t compression);

13
source/client/src/clientEnv.c
View File

@ -35,6 +35,7 @@
#include "tsched.h"
#include "ttime.h"
#include "tversion.h"
#include "tcompare.h"
#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL)
#include "cus_name.h"
@ -875,6 +876,12 @@ void taos_init_imp(void) {
}
rpcInit();
if (InitRegexCache() != 0) {
tscInitRes = -1;
tscError("failed to init regex cache");
return;
}
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
catalogInit(&cfg);
@ -890,7 +897,11 @@ void taos_init_imp(void) {
tscError("failed to init task queue");
return;
}
fmFuncMgtInit();
if (fmFuncMgtInit() != TSDB_CODE_SUCCESS) {
tscInitRes = -1;
tscError("failed to init function manager");
return;
}
nodesInitAllocatorSet();
clientConnRefPool = taosOpenRef(200, destroyTscObj);

2
source/client/src/clientMain.c
View File

@ -28,6 +28,7 @@
#include "tref.h"
#include "trpc.h"
#include "version.h"
#include "tcompare.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -78,6 +79,7 @@ void taos_cleanup(void) {
clientConnRefPool = -1;
taosCloseRef(id);
DestroyRegexCache();
rpcCleanup();
tscDebug("rpc cleanup");

3
source/client/src/clientMonitor.c
View File

@ -801,9 +801,8 @@ int32_t monitorInit() {
}
if (taosMulModeMkDir(tmpSlowLogPath, 0777, true) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tscError("failed to create dir:%s since %s", tmpSlowLogPath, terrstr());
return -1;
return terrno;
}
if (tsem2_init(&monitorSem, 0, 0) != 0) {

21
source/client/src/clientTmq.c
View File

@ -448,12 +448,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse
pOffset.consumerId = tmq->consumerId;
pOffset.offset.val = *offset;
int32_t groupLen = strlen(tmq->groupId);
(void)memcpy(pOffset.offset.subKey, tmq->groupId, groupLen);
pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR;
(void)strcpy(pOffset.offset.subKey + groupLen + 1, pTopicName);
(void)snprintf(pOffset.offset.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopicName);
int32_t len = 0;
int32_t code = 0;
tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code);
@ -1746,11 +1741,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
}
void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int32_t groupLen = strlen(tmq->groupId);
(void)memcpy(pReq->subKey, tmq->groupId, groupLen);
pReq->subKey[groupLen] = TMQ_SEPARATOR;
(void)strcpy(pReq->subKey + groupLen + 1, pTopic->topicName);
(void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName);
pReq->withTbName = tmq->withTbName;
pReq->consumerId = tmq->consumerId;
pReq->timeout = timeout;
@ -2804,7 +2795,7 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
pWrapper->epoch = head->epoch;
(void)memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) != NULL){
if (tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg) == NULL){
taosFreeQitem(pWrapper);
}else{
(void)taosWriteQitem(tmq->mqueue, pWrapper);
@ -3077,11 +3068,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep
SMqVgOffset pOffset = {0};
pOffset.consumerId = tmq->consumerId;
int32_t groupLen = strlen(tmq->groupId);
(void)memcpy(pOffset.offset.subKey, tmq->groupId, groupLen);
pOffset.offset.subKey[groupLen] = TMQ_SEPARATOR;
(void)strcpy(pOffset.offset.subKey + groupLen + 1, tname);
(void)snprintf(pOffset.offset.subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, tname);
int32_t len = 0;
tEncodeSize(tEncodeMqVgOffset, &pOffset, len, code);

3
source/common/src/tglobal.c
View File

@ -756,7 +756,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1800, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
@ -1306,7 +1306,6 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
taosSetAllDebugFlag(pCfg, cfgGetItem(pCfg, "debugFlag")->i32);
if (taosMulModeMkDir(tsLogDir, 0777, true) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
printf("failed to create dir:%s since %s", tsLogDir, terrstr());
cfgCleanup(pCfg);
return -1;

1
source/common/src/tmisce.c
View File

@ -150,7 +150,6 @@ int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else {

14
source/common/src/tmsg.c
View File

@ -9197,23 +9197,23 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
if (pVal->type == TMQ_OFFSET__RESET_NONE) {
snprintf(buf, maxLen, "none");
(void)snprintf(buf, maxLen, "none");
} else if (pVal->type == TMQ_OFFSET__RESET_EARLIEST) {
snprintf(buf, maxLen, "earliest");
(void)snprintf(buf, maxLen, "earliest");
} else if (pVal->type == TMQ_OFFSET__RESET_LATEST) {
snprintf(buf, maxLen, "latest");
(void)snprintf(buf, maxLen, "latest");
} else if (pVal->type == TMQ_OFFSET__LOG) {
snprintf(buf, maxLen, "wal:%" PRId64, pVal->version);
(void)snprintf(buf, maxLen, "wal:%" PRId64, pVal->version);
} else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
if (IS_VAR_DATA_TYPE(pVal->primaryKey.type)) {
char *tmp = taosMemoryCalloc(1, pVal->primaryKey.nData + 1);
if (tmp == NULL) return TSDB_CODE_OUT_OF_MEMORY;
memcpy(tmp, pVal->primaryKey.pData, pVal->primaryKey.nData);
snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%s", pVal->uid, pVal->ts,
(void)memcpy(tmp, pVal->primaryKey.pData, pVal->primaryKey.nData);
(void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%s", pVal->uid, pVal->ts,
pVal->primaryKey.type, tmp);
taosMemoryFree(tmp);
} else {
snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts,
(void)snprintf(buf, maxLen, "tsdb:%" PRId64 "|%" PRId64 ",pk type:%d,val:%" PRId64, pVal->uid, pVal->ts,
pVal->primaryKey.type, pVal->primaryKey.val);
}
} else {

181
source/common/src/ttime.c
View File

@ -35,7 +35,7 @@ void deltaToUtcInitOnce() {
// printf("====delta:%lld\n\n", seconds);
}
static int64_t parseFraction(char* str, char** end, int32_t timePrec);
static int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction);
static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim);
static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim);
@ -95,7 +95,9 @@ char* forwardToTimeStringEnd(char* str) {
return &str[i];
}
int64_t parseFraction(char* str, char** end, int32_t timePrec) {
int32_t parseFraction(char* str, char** end, int32_t timePrec, int64_t* pFraction) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t i = 0;
int64_t fraction = 0;
@ -112,7 +114,7 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
int32_t totalLen = i;
if (totalLen <= 0) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
/* parse the fraction */
@ -134,21 +136,24 @@ int64_t parseFraction(char* str, char** end, int32_t timePrec) {
}
times = NANO_SEC_FRACTION_LEN - i;
} else {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
fraction = strnatoi(str, i) * factor[times];
*end = str + totalLen;
*pFraction = fraction;
return fraction;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t parseTimezone(char* str, int64_t* tzOffset) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t hour = 0;
int32_t i = 0;
if (str[i] != '+' && str[i] != '-') {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
i++;
@ -160,7 +165,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
continue;
}
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
char* sep = strchr(&str[i], ':');
@ -175,18 +180,18 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
}
if (hour > 12 || hour < 0) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
// return error if there're illegal charaters after min(2 Digits)
char* minStr = &str[i];
if (minStr[1] != '\0' && minStr[2] != '\0') {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
int64_t minute = strnatoi(&str[i], 2);
if (minute > 59 || (hour == 12 && minute > 0)) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
if (str[0] == '+') {
@ -195,13 +200,13 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
*tzOffset = hour * 3600 + minute * 60;
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) {
*offset = 0;
return 0;
return TSDB_CODE_SUCCESS;
}
return parseTimezone(tzStr, offset);
}
@ -219,6 +224,8 @@ int32_t offsetOfTimezone(char* tzStr, int64_t* offset) {
* 2013-04-12T15:52:01.123+0800
*/
int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
int64_t factor = TSDB_TICK_PER_SECOND(timePrec);
int64_t tzOffset = 0;
@ -234,7 +241,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
}
if (str == NULL) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
/* mktime will be affected by TZ, set by using taos_options */
@ -253,22 +260,18 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor;
} else if (str[0] == '.') {
str += 1;
if ((fraction = parseFraction(str, &str, timePrec)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(parseFraction(str, &str, timePrec, &fraction));
*time = seconds * factor + fraction;
char seg = str[0];
if (seg != 'Z' && seg != 'z' && seg != '+' && seg != '-') {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if ((seg == 'Z' || seg == 'z') && str[1] != '\0') {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
} else if (seg == '+' || seg == '-') {
// parse the timezone
if (parseTimezone(str, &tzOffset) == -1) {
return -1;
}
TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
*time += tzOffset * factor;
}
@ -277,16 +280,14 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch
*time = seconds * factor + fraction;
// parse the timezone
if (parseTimezone(str, &tzOffset) == -1) {
return -1;
}
TAOS_CHECK_RETURN(parseTimezone(str, &tzOffset));
*time += tzOffset * factor;
} else {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
static FORCE_INLINE bool validateTm(struct tm* pTm) {
@ -314,6 +315,8 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) {
}
int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0;
struct tm tm = {0};
@ -330,7 +333,7 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
// if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
}
@ -347,16 +350,16 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timeP
if (*str == '.') {
/* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
}
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) {
int32_t code = TSDB_CODE_SUCCESS;
*utime = 0;
struct tm tm = {0};
tm.tm_isdst = -1;
@ -374,7 +377,7 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
// if parse failed, try "%Y-%m-%d" format
str = taosStrpTime(timestr, "%Y-%m-%d", &tm);
if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
}
@ -384,13 +387,11 @@ int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t ti
int64_t fraction = 0;
if (*str == '.') {
/* parse the second fraction part */
if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) {
return -1;
}
TAOS_CHECK_RETURN(parseFraction(str + 1, &str, timePrec, &fraction));
}
*utime = TSDB_TICK_PER_SECOND(timePrec) * seconds + fraction;
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
char getPrecisionUnit(int32_t precision) {
@ -482,10 +483,12 @@ int64_t convertTimePrecision(int64_t utime, int32_t fromPrecision, int32_t toPre
// !!!!notice: double lose precison if time is too large, for example: 1626006833631000000*1.0 = double =
// 1626006833631000064
int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
int32_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit, int64_t* pRes) {
int32_t code = TSDB_CODE_SUCCESS;
if (fromPrecision != TSDB_TIME_PRECISION_MILLI && fromPrecision != TSDB_TIME_PRECISION_MICRO &&
fromPrecision != TSDB_TIME_PRECISION_NANO) {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
@ -541,15 +544,23 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
time *= factors[fromPrecision];
break;
default: {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
}
if (tmp >= (double)INT64_MAX) return INT64_MAX;
if (tmp <= (double)INT64_MIN) return INT64_MIN;
return time;
if (tmp >= (double)INT64_MAX) {
*pRes = INT64_MAX;
} else if (tmp <= (double)INT64_MIN) {
*pRes = INT64_MIN;
} else {
*pRes = time;
}
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t charLen = varDataLen(inputData);
char* newColData;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
@ -558,7 +569,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int32_t ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData);
return TSDB_CODE_INVALID_TIMESTAMP;
TAOS_RETURN(TSDB_CODE_INVALID_TIMESTAMP);
}
taosMemoryFree(newColData);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
@ -566,50 +577,52 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
int len = taosUcs4ToMbs((TdUcs4*)varDataVal(inputData), charLen, newColData);
if (len < 0) {
taosMemoryFree(newColData);
return TSDB_CODE_FAILED;
TAOS_RETURN(TSDB_CODE_FAILED);
}
newColData[len] = 0;
int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData);
return ret;
TAOS_RETURN(ret);
}
taosMemoryFree(newColData);
} else {
return TSDB_CODE_FAILED;
TAOS_RETURN(TSDB_CODE_FAILED);
}
return TSDB_CODE_SUCCESS;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
switch (unit) {
case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
(*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision);
break;
case 'm':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
(*result) = convertTimePrecision(val * MILLISECOND_PER_MINUTE, TSDB_TIME_PRECISION_MILLI, timePrecision);
break;
case 'h':
if (val > INT64_MAX / MILLISECOND_PER_MINUTE) {
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
(*result) = convertTimePrecision(val * MILLISECOND_PER_HOUR, TSDB_TIME_PRECISION_MILLI, timePrecision);
break;
case 'd':
if (val > INT64_MAX / MILLISECOND_PER_DAY) {
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
(*result) = convertTimePrecision(val * MILLISECOND_PER_DAY, TSDB_TIME_PRECISION_MILLI, timePrecision);
break;
case 'w':
if (val > INT64_MAX / MILLISECOND_PER_WEEK) {
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
(*result) = convertTimePrecision(val * MILLISECOND_PER_WEEK, TSDB_TIME_PRECISION_MILLI, timePrecision);
break;
@ -623,10 +636,10 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
(*result) = convertTimePrecision(val, TSDB_TIME_PRECISION_NANO, timePrecision);
break;
default: {
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
}
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
/*
@ -645,36 +658,41 @@ int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecisi
*/
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit,
int32_t timePrecision) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0;
char* endPtr = NULL;
/* get the basic numeric value */
int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
if ((timestamp == 0 && token[0] != '0') || errno != 0) {
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
/* natual month/year are not allowed in absolute duration */
*unit = token[tokenlen - 1];
if (*unit == 'n' || *unit == 'y') {
return -1;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
return getDuration(timestamp, *unit, duration, timePrecision);
}
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) {
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision,
bool negativeAllow) {
int32_t code = TSDB_CODE_SUCCESS;
errno = 0;
/* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10);
if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
*unit = token[tokenLen - 1];
if (*unit == 'n' || *unit == 'y') {
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
if (isdigit(*unit)) {
*unit = getPrecisionUnit(timePrecision);
@ -683,9 +701,7 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
return getDuration(*duration, *unit, duration, timePrecision);
}
static bool taosIsLeapYear(int32_t year) {
return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0));
}
static bool taosIsLeapYear(int32_t year) { return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)); }
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
if (duration == 0) {
@ -900,7 +916,8 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) {
// used together with taosTimeTruncate. when offset is great than zero, slide-start/slide-end is the anchor point
int64_t taosTimeGetIntervalEnd(int64_t intervalStart, const SInterval* pInterval) {
if (pInterval->offset > 0) {
int64_t slideStart = taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision);
int64_t slideStart =
taosTimeAdd(intervalStart, -1 * pInterval->offset, pInterval->offsetUnit, pInterval->precision);
int64_t slideEnd = taosTimeAdd(slideStart, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1;
int64_t result = taosTimeAdd(slideEnd, pInterval->offset, pInterval->offsetUnit, pInterval->precision);
return result;
@ -960,7 +977,9 @@ const char* fmtts(int64_t ts) {
return buf;
}
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
int32_t taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) {
int32_t code = TSDB_CODE_SUCCESS;
char ts[40] = {0};
struct tm ptm;
@ -996,17 +1015,18 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
default:
fractionLen = 0;
return;
TAOS_RETURN(TSDB_CODE_INVALID_PARA);
}
if (taosLocalTime(&quot, &ptm, buf) == NULL) {
return;
TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
}
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm);
length += snprintf(ts + length, fractionLen, format, mod);
length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm);
tstrncpy(buf, ts, bufLen);
TAOS_RETURN(TSDB_CODE_SUCCESS);
}
int32_t taosTs2Tm(int64_t ts, int32_t precision, struct STm* tm) {
@ -1923,8 +1943,8 @@ int32_t taosTs2Char(const char* format, SArray** formats, int64_t ts, int32_t pr
return tm2char(*formats, &tm, out, outLen);
}
int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision, char* errMsg,
int32_t errMsgLen) {
int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int64_t* ts, int32_t precision,
char* errMsg, int32_t errMsgLen) {
const char* sErrPos;
int32_t fErrIdx;
if (!*formats) {
@ -1989,7 +2009,8 @@ static int64_t UNIT_MATRIX[10][11] = { /* ns, us, ms, s, min, h,
/*mon*/ {0, 0, 0, 0, 0, 0, 0, 0, 1, 12, 0},
/*y*/ {0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0}};
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) {
static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) {
if (interval < baseInterval) return false;
@ -2021,7 +2042,8 @@ static bool recursiveTsmaCheckRecursive(int64_t baseInterval, int8_t baseIdx, in
return false;
}
static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx, bool checkEq) {
static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t baseIdx, int64_t interval, int8_t idx,
bool checkEq) {
if (UNIT_MATRIX[baseIdx][idx] == -1) return false;
if (baseIdx == idx) {
@ -2041,18 +2063,27 @@ static bool recursiveTsmaCheckRecursiveReverse(int64_t baseInterval, int8_t base
/*
* @breif check if tsma with param [interval], [unit] can create based on base tsma with baseInterval and baseUnit
* @param baseInterval, baseUnit, interval/unit of base tsma
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already been
* translated to TICKS of [precision]
* @param interval the tsma interval going to create. Not that if unit is not calander unit, then interval has already
* been translated to TICKS of [precision]
* @param unit the tsma unit going to create
* @param precision the precision of this db
* @param checkEq pass true if same interval is not acceptable, false if acceptable.
* @ret true the tsma can be created, else cannot
* */
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision, bool checkEq) {
bool checkRecursiveTsmaInterval(int64_t baseInterval, int8_t baseUnit, int64_t interval, int8_t unit, int8_t precision,
bool checkEq) {
bool baseIsCalendarDuration = IS_CALENDAR_TIME_DURATION(baseUnit);
if (!baseIsCalendarDuration) baseInterval = convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit);
if (!baseIsCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(baseInterval, precision, baseUnit, &baseInterval)) {
return false;
}
}
bool isCalendarDuration = IS_CALENDAR_TIME_DURATION(unit);
if (!isCalendarDuration) interval = convertTimeFromPrecisionToUnit(interval, precision, unit);
if (!isCalendarDuration) {
if (TSDB_CODE_SUCCESS != convertTimeFromPrecisionToUnit(interval, precision, unit, &interval)) {
return false;
}
}
bool needCheckEq = baseIsCalendarDuration == isCalendarDuration && checkEq;

9
source/dnode/mgmt/mgmt_qnode/src/qmInt.c
View File

@ -34,11 +34,8 @@ static void qmClose(SQnodeMgmt *pMgmt) {
}
static int32_t qndOpenWrapper(SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = qndOpen(pOption);
if (*pQnode == NULL) {
return terrno;
}
return 0;
int32_t code = qndOpen(pOption, pQnode);
return code;
}
static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = 0;
@ -62,7 +59,7 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
if (code != 0) {
dError("failed to open qnode since %s", tstrerror(code));
qmClose(pMgmt);
return -1;
return code;
}
tmsgReportStartup("qnode-impl", "initialized");

4
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -48,8 +48,8 @@ static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
}
int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
qndPreprocessQueryMsg(pMgmt->pQnode, pMsg);
int32_t code = qndPreprocessQueryMsg(pMgmt->pQnode, pMsg);
if (code) return code;
return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);
}

4
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -262,7 +262,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
dmReleaseWrapper(pWrapper);
code = TAOS_SYSTEM_ERROR(errno);
code = terrno;
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
}
@ -333,7 +333,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex);
code = TAOS_SYSTEM_ERROR(errno);
code = terrno;
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
}

2
source/dnode/mgmt/node_mgmt/src/dmNodes.c
View File

@ -21,7 +21,7 @@ int32_t dmOpenNode(SMgmtWrapper *pWrapper) {
SDnode *pDnode = pWrapper->pDnode;
if (taosMkDir(pWrapper->path) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
code = terrno;
dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, tstrerror(code));
return code;
}

2
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -417,7 +417,7 @@ int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
}
if (taosMkDir(folder) != 0) {
code = TAOS_SYSTEM_ERROR(errno);
code = terrno;
encryptError("failed to create dir:%s since %s", folder, tstrerror(code));
goto _OVER;
}

20
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -30,9 +30,9 @@ enum {
int32_t mndInitConsumer(SMnode *pMnode);
void mndCleanupConsumer(SMnode *pMnode);
void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info);
int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo* info);
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId);
int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer);
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
@ -43,6 +43,22 @@ int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer);
const char *mndConsumerStatusName(int status);
#define MND_TMQ_NULL_CHECK(c) \
do { \
if (c == NULL) { \
code = TSDB_CODE_OUT_OF_MEMORY; \
goto END; \
} \
} while (0)
#define MND_TMQ_RETURN_CHECK(c) \
do { \
code = c; \
if (code != 0) { \
goto END; \
} \
} while (0)
#ifdef __cplusplus
}
#endif

7
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -622,7 +622,8 @@ typedef struct {
int32_t resetOffsetCfg;
} SMqConsumerObj;
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe);
int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType,
char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer);
void tClearSMqConsumerObj(SMqConsumerObj* pConsumer);
void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer);
int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
@ -665,8 +666,8 @@ typedef struct {
char* qmsg; // SubPlanToString
} SMqSubscribeObj;
SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub);
int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub);
int32_t tCloneSubscribeObj(const SMqSubscribeObj* pSub, SMqSubscribeObj **ppSub);
void tDeleteSubscribeObj(SMqSubscribeObj* pSub);
int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub);
void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver);

17
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -98,7 +98,7 @@ typedef struct STaskChkptInfo {
int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode);
SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName);
int32_t mndAcquireStream(SMnode *pMnode, char *streamName, SStreamObj **pStream);
void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream);
@ -109,17 +109,18 @@ int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId);
int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams);
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg);
int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg, STrans **pTrans1);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId);
int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj** pStream);
int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
int32_t mndProcessStreamHb(SRpcMsg *pReq);
void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
@ -137,16 +138,16 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStrea
int64_t ts);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
int32_t createStreamTaskIter(SStreamObj *pStream, SStreamTaskIter **pIter);
void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
void mndInitExecInfo();
int32_t streamTaskIterGetCurrent(SStreamTaskIter *pIter, SStreamTask **pTask);
int32_t mndInitExecInfo();
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
SCheckpointConsensusInfo *mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks);
int32_t mndGetConsensusInfo(SHashObj *pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo);
void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpointInfo *pRestoreInfo);
void mndClearConsensusRspEntry(SCheckpointConsensusInfo *pInfo);
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId);

5
source/dnode/mnode/impl/inc/mndSubscribe.h
View File

@ -26,12 +26,9 @@ int32_t mndInitSubscribe(SMnode *pMnode);
void mndCleanupSubscribe(SMnode *pMnode);
int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName);
SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, const char *CGroup, const char *topicName);
SMqSubscribeObj *mndAcquireSubscribeByKey(SMnode *pMnode, const char *key);
int32_t mndAcquireSubscribeByKey(SMnode *pMnode, const char *key, SMqSubscribeObj** pSub);
void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
void mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName);
int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topic);
int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub);

5
source/dnode/mnode/impl/inc/mndTopic.h
View File

@ -25,13 +25,12 @@ extern "C" {
int32_t mndInitTopic(SMnode *pMnode);
void mndCleanupTopic(SMnode *pMnode);
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName);
int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic);
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb);
const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
void mndTopicGetShowName(const char* fullTopic, char* topic);
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics);
#ifdef __cplusplus

506
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -40,7 +40,6 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg);
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg);
static int32_t mndProcessMqHbReq(SRpcMsg *pMsg);
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg);
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg);
int32_t mndInitConsumer(SMnode *pMnode) {
SSdbTable table = {
@ -57,7 +56,6 @@ int32_t mndInitConsumer(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq);
// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg);
mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer);
@ -68,13 +66,10 @@ int32_t mndInitConsumer(SMnode *pMnode) {
void mndCleanupConsumer(SMnode *pMnode) {}
void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) {
int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) {
int32_t code = 0;
void *msg = rpcMallocCont(sizeof(int64_t));
if (msg == NULL) {
mError("consumer:0x%" PRIx64 " failed to clear consumer due to out of memory. alloc size:%d", consumerId,
(int32_t)sizeof(int64_t));
return;
}
MND_TMQ_NULL_CHECK(msg);
*(int64_t*)msg = consumerId;
SRpcMsg rpcMsg = {
@ -85,108 +80,51 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR
};
mInfo("mndSendConsumerMsg type:%d consumer:0x%" PRIx64, msgType, consumerId);
int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
if (code != 0){
mError("consumer:%"PRId64" send consumer msg:%d error:%d", consumerId, msgType, code);
}
MND_TMQ_RETURN_CHECK(tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg));
return code;
END:
taosMemoryFree(msg);
return code;
}
static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser,
bool enableReplay) {
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
int32_t lino = 0;
int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) {
char *pOneTopic = taosArrayGetP(pTopicList, i);
pTopic = mndAcquireTopic(pMnode, pOneTopic);
if (pTopic == NULL) { // terrno has been set by callee function
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED;
}
TAOS_CHECK_GOTO(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic), &lino, FAILED);
TAOS_CHECK_GOTO(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION), &lino, FAILED);
MND_TMQ_RETURN_CHECK(mndAcquireTopic(pMnode, pOneTopic, &pTopic));
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic));
MND_TMQ_RETURN_CHECK(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION));
if (enableReplay) {
if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT;
goto FAILED;
goto END;
} else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
if (pDb == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED;
goto END;
}
if (pDb->cfg.numOfVgroups != 1) {
mndReleaseDb(pMnode, pDb);
code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP;
goto FAILED;
goto END;
}
mndReleaseDb(pMnode, pDb);
}
}
mndReleaseTopic(pMnode, pTopic);
}
return 0;
FAILED:
mndReleaseTopic(pMnode, pTopic);
TAOS_RETURN(code);
}
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
if (pConsumer == NULL) {
mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId,
pConsumer->status, mndConsumerStatusName(pConsumer->status));
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) {
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto END;
}
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL);
if (pConsumerNew == NULL){
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
TAOS_CHECK_GOTO(validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false), &lino, END);
TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), &lino, END);
code = mndTransPrepare(pMnode, pTrans);
END:
mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans);
TAOS_RETURN(code);
mndReleaseTopic(pMnode, pTopic);
return code;
}
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
@ -196,55 +134,39 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
SMqConsumerClearMsg *pClearMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL;
SMqConsumerObj *pConsumer = NULL;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId);
TAOS_RETURN(code);
}
MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, pClearMsg->consumerId, &pConsumer));
mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId,
mndConsumerStatusName(pConsumer->status));
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL);
if (pConsumerNew == NULL){
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL, &pConsumerNew));
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END;
}
// this is the drop action, not the update action
TAOS_CHECK_GOTO(mndSetConsumerDropLogs(pTrans, pConsumerNew), &lino, END);
MND_TMQ_NULL_CHECK(pTrans);
MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumerNew));
code = mndTransPrepare(pMnode, pTrans);
END:
mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans);
TAOS_RETURN(code);
return code;
}
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) {
int32_t code = 0;
rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege));
if (rsp->topicPrivileges == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
MND_TMQ_NULL_CHECK(rsp->topicPrivileges);
for (int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if (pTopic == NULL) { // terrno has been set by callee function
SMqTopicObj *pTopic = NULL;
code = mndAcquireTopic(pMnode, topic, &pTopic);
if (code != TDB_CODE_SUCCESS) {
continue;
}
STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1);
strcpy(data->topic, topic);
MND_TMQ_NULL_CHECK(data);
(void)strcpy(data->topic, topic);
if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 ||
grantCheckExpire(TSDB_GRANT_SUBSCRIPTION) < 0) {
data->noPrivilege = 1;
@ -253,22 +175,30 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRs
}
mndReleaseTopic(pMnode, pTopic);
}
return 0;
END:
return code;
}
static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pConsumer){
for (int i = 0; i < taosArrayGetSize(req->topics); i++) {
TopicOffsetRows *data = taosArrayGet(req->topics, i);
if (data == NULL){
continue;
}
mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName);
if (pSub == NULL) {
SMqSubscribeObj *pSub = NULL;
char key[TSDB_SUBSCRIBE_KEY_LEN] = {0};
(void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, data->topicName);
int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub);
if (code != 0) {
mError("failed to acquire subscribe by key:%s, code:%d", key, code);
continue;
}
taosWLockLatch(&pSub->lock);
SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t));
if (pConsumerEp) {
taosArrayDestroy(pConsumerEp->offsetRows);
(void)taosArrayDestroy(pConsumerEp->offsetRows);
pConsumerEp->offsetRows = data->offsetRows;
data->offsetRows = NULL;
}
@ -303,36 +233,24 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
SMqHbReq req = {0};
SMqHbRsp rsp = {0};
SMqConsumerObj *pConsumer = NULL;
TAOS_CHECK_GOTO(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req), NULL, end);
MND_TMQ_RETURN_CHECK(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req));
int64_t consumerId = req.consumerId;
pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " not exist", consumerId);
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto end;
}
TAOS_CHECK_GOTO(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user), NULL, end);
MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user));
atomic_store_32(&pConsumer->hbStatus, 0);
int32_t status = atomic_load_32(&pConsumer->status);
if (status == MQ_CONSUMER_STATUS_LOST) {
mInfo("try to recover consumer:0x%" PRIx64, consumerId);
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info);
MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info));
}
storeOffsetRows(pMnode, &req, pConsumer);
code = buildMqHbRsp(pMsg, &rsp);
end:
END:
tDestroySMqHbRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer);
tDestroySMqHbReq(&req);
TAOS_RETURN(code);
return code;
}
static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){
@ -349,19 +267,22 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
// handle all topics subscribed by this consumer
for (int32_t i = 0; i < numOfTopics; i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
// txn guarantees pSub is created
if (pSub == NULL) {
SMqSubscribeObj *pSub = NULL;
char key[TSDB_SUBSCRIBE_KEY_LEN] = {0};
(void)snprintf(key, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", pConsumer->cgroup, TMQ_SEPARATOR, topic);
int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub);
if (code != 0) {
continue;
}
taosRLockLatch(&pSub->lock);
SMqSubTopicEp topicEp = {0};
strcpy(topicEp.topic, topic);
(void)strcpy(topicEp.topic, topic);
// 2.1 fetch topic schema
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
if (pTopic == NULL) {
SMqTopicObj *pTopic = NULL;
code = mndAcquireTopic(pMnode, topic, &pTopic);
if (code != TDB_CODE_SUCCESS) {
taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub);
continue;
@ -371,16 +292,27 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
topicEp.schema.nCols = pTopic->schema.nCols;
if (topicEp.schema.nCols) {
topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema));
memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema));
if (topicEp.schema.pSchema == NULL) {
taosRUnLockLatch(&pTopic->lock);
taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub);
mndReleaseTopic(pMnode, pTopic);
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(topicEp.schema.pSchema, pTopic->schema.pSchema, topicEp.schema.nCols * sizeof(SSchema));
}
taosRUnLockLatch(&pTopic->lock);
mndReleaseTopic(pMnode, pTopic);
// 2.2 iterate all vg assigned to the consumer of that topic
SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &pConsumer->consumerId, sizeof(int64_t));
if (pConsumerEp == NULL) {
taosRUnLockLatch(&pConsumer->lock);
taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub);
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs);
// this customer assigned vgroups
topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
if (topicEp.vgs == NULL) {
taosRUnLockLatch(&pConsumer->lock);
@ -391,9 +323,9 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
// char offsetKey[TSDB_PARTITION_KEY_LEN];
// mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
if (pVgEp == NULL) {
continue;
}
if (epoch == -1) {
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (pVgroup) {
@ -401,17 +333,10 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
mndReleaseVgroup(pMnode, pVgroup);
}
}
// 2.2.1 build vg ep
SMqSubVgEp vgEp = {
.epSet = pVgEp->epSet,
.vgId = pVgEp->vgId,
.offset = -1,
};
taosArrayPush(topicEp.vgs, &vgEp);
SMqSubVgEp vgEp = {.epSet = pVgEp->epSet, .vgId = pVgEp->vgId, .offset = -1};
(void)taosArrayPush(topicEp.vgs, &vgEp);
}
taosArrayPush(rsp->topics, &topicEp);
(void)taosArrayPush(rsp->topics, &topicEp);
taosRUnLockLatch(&pSub->lock);
mndReleaseSubscribe(pMnode, pSub);
}
@ -437,12 +362,15 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc
pHead->walever = 0;
void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqAskEpRsp(&abuf, rsp);
if (tEncodeSMqAskEpRsp(&abuf, rsp) < 0) {
rpcFreeCont(buf);
return TSDB_CODE_TSC_INTERNAL_ERROR;
}
// send rsp
pMsg->info.rsp = buf;
pMsg->info.rspLen = tlen;
TAOS_RETURN(code);
return code;
}
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
@ -450,33 +378,22 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
SMqAskEpReq req = {0};
SMqAskEpRsp rsp = {0};
int32_t code = 0;
SMqConsumerObj *pConsumer = NULL;
TAOS_CHECK_RETURN(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req));
MND_TMQ_RETURN_CHECK(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req));
int64_t consumerId = req.consumerId;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " group:%s not exists in sdb", consumerId, req.cgroup);
return TSDB_CODE_MND_CONSUMER_NOT_EXIST;
}
MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
if (strncmp(req.cgroup, pConsumer->cgroup, tListLen(pConsumer->cgroup)) != 0) {
mError("consumer:0x%" PRIx64 " group:%s not consistent with data in sdb, saved cgroup:%s", consumerId, req.cgroup,
pConsumer->cgroup);
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto END;
}
atomic_store_32(&pConsumer->hbStatus, 0);
// 1. check consumer status
int32_t status = atomic_load_32(&pConsumer->status);
if (status == MQ_CONSUMER_STATUS_LOST) {
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info);
MND_TMQ_RETURN_CHECK(mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_CONSUMER_RECOVER, &pMsg->info));
}
if (status != MQ_CONSUMER_STATUS_READY) {
mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status));
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
@ -490,10 +407,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
if (epoch != serverEpoch) {
mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d",
consumerId, epoch, serverEpoch);
code = addEpSetInfo(pMnode, pConsumer, epoch, &rsp);
if(code != 0){
goto END;
}
MND_TMQ_RETURN_CHECK(addEpSetInfo(pMnode, pConsumer, epoch, &rsp));
}
code = buildAskEpRsp(pMsg, &rsp, serverEpoch, consumerId);
@ -507,27 +421,29 @@ END:
int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
END:
return code;
}
int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if (code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
END:
return code;
}
static void freeItem(void *param) {
@ -538,29 +454,35 @@ static void freeItem(void *param) {
}
static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerObj *pConsumerNew){
int32_t code = 0;
pConsumerNew->rebNewTopics = taosArrayInit(0, sizeof(void *));
MND_TMQ_NULL_CHECK(pConsumerNew->rebNewTopics);
pConsumerNew->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
if(pConsumerNew->rebNewTopics == NULL || pConsumerNew->rebRemovedTopics == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
MND_TMQ_NULL_CHECK(pConsumerNew->rebRemovedTopics);
int32_t newTopicNum = taosArrayGetSize(pConsumerNew->assignedTopics);
int32_t oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics);
int32_t i = 0, j = 0;
while (i < oldTopicNum || j < newTopicNum) {
if (i >= oldTopicNum) {
char *newTopicCopy = taosStrdup(taosArrayGetP(pConsumerNew->assignedTopics, j));
taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
void* tmp = taosArrayGetP(pConsumerNew->assignedTopics, j);
MND_TMQ_NULL_CHECK(tmp);
char *newTopicCopy = taosStrdup(tmp);
MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy));
j++;
continue;
} else if (j >= newTopicNum) {
char *oldTopicCopy = taosStrdup(taosArrayGetP(pExistedConsumer->currentTopics, i));
taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
void* tmp = taosArrayGetP(pExistedConsumer->currentTopics, i);
MND_TMQ_NULL_CHECK(tmp);
char *oldTopicCopy = taosStrdup(tmp);
MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy));
i++;
continue;
} else {
char *oldTopic = taosArrayGetP(pExistedConsumer->currentTopics, i);
MND_TMQ_NULL_CHECK(oldTopic);
char *newTopic = taosArrayGetP(pConsumerNew->assignedTopics, j);
MND_TMQ_NULL_CHECK(newTopic);
int comp = strcmp(oldTopic, newTopic);
if (comp == 0) {
i++;
@ -568,22 +490,22 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb
continue;
} else if (comp < 0) {
char *oldTopicCopy = taosStrdup(oldTopic);
taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy);
MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy));
i++;
continue;
} else {
char *newTopicCopy = taosStrdup(newTopic);
taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy);
MND_TMQ_NULL_CHECK(taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy));
j++;
continue;
}
}
}
return 0;
END:
return code;
}
static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
int32_t code = 0;
taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
@ -594,24 +516,21 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
}
}
TAOS_RETURN(code);
return 0;
}
static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){
static int32_t buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer){
int64_t consumerId = subscribe->consumerId;
char *cgroup = subscribe->cgroup;
SMqConsumerObj *pConsumerNew = NULL;
SMqConsumerObj *pExistedConsumer = NULL;
pExistedConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pExistedConsumer == NULL) {
int32_t code = mndAcquireConsumer(pMnode, consumerId, &pExistedConsumer);
if (code != 0) {
mInfo("receive subscribe request from new consumer:0x%" PRIx64
",cgroup:%s, numOfTopics:%d", consumerId,
subscribe->cgroup, (int32_t)taosArrayGetSize(subscribe->topicNames));
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_INSERT_SUB, NULL, subscribe);
if (pConsumerNew == NULL) {
goto _over;
}
MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_INSERT_SUB, NULL, subscribe, &pConsumerNew));
} else {
int32_t status = atomic_load_32(&pExistedConsumer->status);
@ -621,82 +540,53 @@ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscri
(int32_t)taosArrayGetSize(subscribe->topicNames));
if (status != MQ_CONSUMER_STATUS_READY) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto _over;
}
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_UPDATE_SUB, NULL, subscribe);
if (pConsumerNew == NULL) {
goto _over;
}
int32_t code = getTopicAddDelete(pExistedConsumer, pConsumerNew);
if (code != 0){
terrno = code;
goto _over;
code = TSDB_CODE_MND_CONSUMER_NOT_READY;
goto END;
}
MND_TMQ_RETURN_CHECK(tNewSMqConsumerObj(consumerId, cgroup, CONSUMER_UPDATE_SUB, NULL, subscribe, &pConsumerNew));
MND_TMQ_RETURN_CHECK(getTopicAddDelete(pExistedConsumer, pConsumerNew));
}
mndReleaseConsumer(pMnode, pExistedConsumer);
return pConsumerNew;
if (ppConsumer){
*ppConsumer = pConsumerNew;
}
return code;
_over:
END:
mndReleaseConsumer(pMnode, pExistedConsumer);
tDeleteSMqConsumerObj(pConsumerNew);
return NULL;
return code;
}
int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node;
char *msgStr = pMsg->pCont;
int32_t code = 0;
SCMSubscribeReq subscribe = {0};
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL;
SCMSubscribeReq subscribe = {0};
MND_TMQ_RETURN_CHECK(tDeserializeSCMSubscribeReq(msgStr, &subscribe));
if(taosArrayGetSize(subscribe.topicNames) == 0){
SMqConsumerObj *pConsumerTmp = mndAcquireConsumer(pMnode, subscribe.consumerId);
if(pConsumerTmp == NULL){
goto _over;
}
SMqConsumerObj *pConsumerTmp = NULL;
MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp));
mndReleaseConsumer(pMnode, pConsumerTmp);
}
code = checkAndSortTopic(pMnode, subscribe.topicNames);
if(code != TSDB_CODE_SUCCESS){
goto _over;
}
MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames));
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over;
}
MND_TMQ_NULL_CHECK(pTrans);
code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
if (code != TSDB_CODE_SUCCESS) {
goto _over;
}
pConsumerNew = buildSubConsumer(pMnode, &subscribe);
if(pConsumerNew == NULL){
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over;
}
TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), NULL, _over);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _over);
MND_TMQ_RETURN_CHECK(validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay));
MND_TMQ_RETURN_CHECK(buildSubConsumer(pMnode, &subscribe, &pConsumerNew));
MND_TMQ_RETURN_CHECK(mndSetConsumerCommitLogs(pTrans, pConsumerNew));
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
code = TSDB_CODE_ACTION_IN_PROGRESS;
_over:
END:
mndTransDrop(pTrans);
tDeleteSMqConsumerObj(pConsumerNew);
taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
TAOS_RETURN(code);
return code;
}
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
@ -713,7 +603,9 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
if (buf == NULL) goto CM_ENCODE_OVER;
void *abuf = buf;
tEncodeSMqConsumerObj(&abuf, pConsumer);
if(tEncodeSMqConsumerObj(&abuf, pConsumer) < 0){
goto CM_ENCODE_OVER;
}
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER);
@ -838,8 +730,7 @@ static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char
int32_t size = taosArrayGetSize(pConsumer->currentTopics);
for (int32_t i = 0; i < size; i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i);
if (strcmp(topic, pTopic) == 0) {
if (topic && strcmp(topic, pTopic) == 0) {
existing = true;
break;
}
@ -865,32 +756,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
} else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) {
int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) {
char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
taosArrayPush(pOldConsumer->rebNewTopics, &topic);
void * tmp = taosArrayGetP(pOldConsumer->assignedTopics, i);
if (tmp == NULL){
return TSDB_CODE_TMQ_INVALID_MSG;
}
char *topic = taosStrdup(tmp);
if (taosArrayPush(pOldConsumer->rebNewTopics, &topic) == NULL) {
taosMemoryFree(topic);
return TSDB_CODE_TMQ_INVALID_MSG;
}
}
pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE;
mInfo("consumer:0x%" PRIx64 " recover update", pOldConsumer->consumerId);
} else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) {
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
(void)atomic_add_fetch_32(&pOldConsumer->epoch, 1);
pOldConsumer->rebalanceTime = taosGetTimestampMs();
mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update, only rebalance time", pOldConsumer->consumerId);
} else if (pNewConsumer->updateType == CONSUMER_ADD_REB) {
char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
void *tmp = taosArrayGetP(pNewConsumer->rebNewTopics, 0);
if (tmp == NULL){
return TSDB_CODE_TMQ_INVALID_MSG;
}
char *pNewTopic = taosStrdup(tmp);
removeFromTopicList(pOldConsumer->rebNewTopics, pNewTopic, pOldConsumer->consumerId, "new");
bool existing = existInCurrentTopicList(pOldConsumer, pNewTopic);
if (existing) {
mError("[rebalance] consumer:0x%" PRIx64 " add new topic:%s should not in currentTopics", pOldConsumer->consumerId, pNewTopic);
taosMemoryFree(pNewTopic);
} else {
taosArrayPush(pOldConsumer->currentTopics, &pNewTopic);
if (taosArrayPush(pOldConsumer->currentTopics, &pNewTopic) == NULL) {
taosMemoryFree(pNewTopic);
return TSDB_CODE_TMQ_INVALID_MSG;
}
taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString);
}
int32_t status = pOldConsumer->status;
updateConsumerStatus(pOldConsumer);
pOldConsumer->rebalanceTime = taosGetTimestampMs();
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
(void)atomic_add_fetch_32(&pOldConsumer->epoch, 1);
mInfo("[rebalance] consumer:0x%" PRIx64 " rebalance update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
", current topics:%d, newTopics:%d, removeTopics:%d",
@ -901,13 +805,16 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
} else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) {
char *topic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
if (topic == NULL){
return TSDB_CODE_TMQ_INVALID_MSG;
}
removeFromTopicList(pOldConsumer->rebRemovedTopics, topic, pOldConsumer->consumerId, "remove");
removeFromTopicList(pOldConsumer->currentTopics, topic, pOldConsumer->consumerId, "current");
int32_t status = pOldConsumer->status;
updateConsumerStatus(pOldConsumer);
pOldConsumer->rebalanceTime = taosGetTimestampMs();
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
(void)atomic_add_fetch_32(&pOldConsumer->epoch, 1);
mInfo("[rebalance]consumer:0x%" PRIx64 " rebalance update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64
", current topics:%d, newTopics:%d, removeTopics:%d",
@ -921,13 +828,13 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
return 0;
}
SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) {
int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer) {
SSdb *pSdb = pMnode->pSdb;
SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId);
if (pConsumer == NULL) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
*pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId);
if (*pConsumer == NULL) {
return TSDB_CODE_MND_CONSUMER_NOT_EXIST;
}
return pConsumer;
return 0;
}
void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
@ -940,6 +847,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SMqConsumerObj *pConsumer = NULL;
int32_t code = 0;
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
@ -964,34 +872,37 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
}
if (numOfRows + topicSz > rowsCapacity) {
blockDataEnsureCapacity(pBlock, numOfRows + topicSz);
MND_TMQ_RETURN_CHECK(blockDataEnsureCapacity(pBlock, numOfRows + topicSz));
}
for (int32_t i = 0; i < topicSz; i++) {
SColumnInfoData *pColInfo;
SColumnInfoData *pColInfo = NULL;
int32_t cols = 0;
// consumer id
char consumerIdHex[32] = {0};
sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId);
(void)sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId);
varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)consumerIdHex, false));
// consumer group
char cgroup[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(cgroup, pConsumer->cgroup);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)cgroup, false));
// client id
char clientId[256 + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(clientId, pConsumer->clientId);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false));
// status
char status[20 + VARSTR_HEADER_SIZE] = {0};
@ -999,46 +910,48 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
STR_TO_VARSTR(status, pStatusName);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)status, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)status, false));
// one subscribed topic
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
MND_TMQ_NULL_CHECK(pColInfo);
if (hasTopic) {
char topic[TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
const char *topicName = mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i));
STR_TO_VARSTR(topic, topicName);
colDataSetVal(pColInfo, numOfRows, (const char *)topic, false);
mndTopicGetShowName(taosArrayGetP(pConsumer->assignedTopics, i), topic + VARSTR_HEADER_SIZE);
*(VarDataLenT *)(topic) = strlen(topic + VARSTR_HEADER_SIZE);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)topic, false));
} else {
colDataSetVal(pColInfo, numOfRows, NULL, true);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, NULL, true));
}
// end point
/*pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);*/
/*colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/
// up time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->createTime, false));
// subscribe time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false));
// rebalance time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0));
char buf[TSDB_OFFSET_LEN] = {0};
STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg};
tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal);
MND_TMQ_RETURN_CHECK(tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal));
char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0};
sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName,
(void)sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName,
pConsumer->autoCommit, pConsumer->autoCommitInterval, buf);
varDataSetLen(parasStr, strlen(varDataVal(parasStr)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)parasStr, false));
numOfRows++;
}
@ -1051,6 +964,9 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *
pShow->numOfRows += numOfRows;
return numOfRows;
END:
return code;
}
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {

237
source/dnode/mnode/impl/src/mndDef.c
View File

@ -12,173 +12,179 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "mndDef.h"
#include "mndConsumer.h"
#include "taoserror.h"
static void *freeStreamTasks(SArray *pTaskLevel);
int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
TAOS_CHECK_RETURN(tStartEncode(pEncoder));
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->name));
if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->totalLevel) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->createTime));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->updateTime));
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->version));
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->totalLevel));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->smaId));
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->uid));
TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->status));
if (tEncodeI8(pEncoder, pObj->conf.igExpired) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->conf.trigger) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->conf.fillHistory) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->conf.triggerParam) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->conf.watermark) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.igExpired));
TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.trigger));
TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->conf.fillHistory));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.triggerParam));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->conf.watermark));
if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->sourceDbUid));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetDbUid));
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sourceDb));
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetDb));
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->targetSTbName));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->targetStbUid));
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, pObj->fixedSinkVgId));
if (pObj->sql != NULL) {
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->sql));
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
}
if (pObj->ast != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->ast));
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
}
if (pObj->physicalPlan != NULL) {
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pObj->physicalPlan));
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, ""));
}
int32_t sz = taosArrayGetSize(pObj->tasks);
if (tEncodeI32(pEncoder, sz) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, sz));
for (int32_t i = 0; i < sz; i++) {
SArray *pArray = taosArrayGetP(pObj->tasks, i);
int32_t innerSz = taosArrayGetSize(pArray);
if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI32(pEncoder, innerSz));
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j);
if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){
pTask->ver = SSTREAM_TASK_VER;
}
if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeStreamTask(pEncoder, pTask));
}
}
if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema));
// 3.0.20 ver =2
if (tEncodeI64(pEncoder, pObj->checkpointFreq) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->igCheckUpdate) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointFreq));
TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->igCheckUpdate));
// 3.0.50 ver = 3
if (tEncodeI64(pEncoder, pObj->checkpointId) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->subTableWithoutMd5) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pObj->checkpointId));
TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pObj->subTableWithoutMd5));
if (tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStrWithLen(pEncoder, pObj->reserve, sizeof(pObj->reserve) - 1));
tEndEncode(pEncoder);
return pEncoder->pos;
}
int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
int32_t code = 0;
TAOS_CHECK_RETURN(tStartDecode(pDecoder));
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->name));
if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->totalLevel) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->createTime));
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->updateTime));
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->version));
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->totalLevel));
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->smaId));
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->uid));
TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->status));
if (tDecodeI8(pDecoder, &pObj->conf.igExpired) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->conf.trigger) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->conf.fillHistory) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->conf.triggerParam) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->conf.watermark) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.igExpired));
TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.trigger));
TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->conf.fillHistory));
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.triggerParam));
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->conf.watermark));
if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->sourceDbUid));
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetDbUid));
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->sourceDb));
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetDb));
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->targetSTbName));
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->targetStbUid));
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &pObj->fixedSinkVgId));
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->ast) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->sql));
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->ast));
TAOS_CHECK_RETURN(tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan));
if (pObj->tasks != NULL) {
pObj->tasks = freeStreamTasks(pObj->tasks);
}
int32_t sz;
if (tDecodeI32(pDecoder, &sz) < 0) {
return -1;
}
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &sz));
if (sz != 0) {
pObj->tasks = taosArrayInit(sz, sizeof(void *));
if (pObj->tasks == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < sz; i++) {
int32_t innerSz;
if (tDecodeI32(pDecoder, &innerSz) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32(pDecoder, &innerSz));
SArray *pArray = taosArrayInit(innerSz, sizeof(void *));
if (pArray != NULL) {
for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) {
taosArrayDestroy(pArray);
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
if (tDecodeStreamTask(pDecoder, pTask) < 0) {
if ((code = tDecodeStreamTask(pDecoder, pTask)) < 0) {
taosMemoryFree(pTask);
taosArrayDestroy(pArray);
return -1;
TAOS_RETURN(code);
}
taosArrayPush(pArray, &pTask);
}
}
taosArrayPush(pObj->tasks, &pArray);
}
}
if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema));
// 3.0.20
if (sver >= 2) {
if (tDecodeI64(pDecoder, &pObj->checkpointFreq) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointFreq));
if (!tDecodeIsEnd(pDecoder)) {
if (tDecodeI8(pDecoder, &pObj->igCheckUpdate) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->igCheckUpdate));
}
}
if (sver >= 3) {
if (tDecodeI64(pDecoder, &pObj->checkpointId) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pDecoder, &pObj->checkpointId));
}
if (sver >= 5) {
if (tDecodeI8(pDecoder, &pObj->subTableWithoutMd5) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI8(pDecoder, &pObj->subTableWithoutMd5));
}
if (tDecodeCStrTo(pDecoder, pObj->reserve) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStrTo(pDecoder, pObj->reserve));
tEndDecode(pDecoder);
return 0;
TAOS_RETURN(code);
}
void *freeStreamTasks(SArray *pTaskLevel) {
@ -220,7 +226,10 @@ void tFreeStreamObj(SStreamObj *pStream) {
SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEpNew == NULL) return NULL;
if (pVgEpNew == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pVgEpNew->vgId = pVgEp->vgId;
// pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
pVgEpNew->epSet = pVgEp->epSet;
@ -255,15 +264,17 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) {
static void *topicNameDup(void *p) { return taosStrdup((char *)p); }
SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType, char *topic, SCMSubscribeReq *subscribe) {
int32_t tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t updateType,
char *topic, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer) {
int32_t code = 0;
SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
if (pConsumer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
pConsumer->consumerId = consumerId;
memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN);
(void)memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN);
pConsumer->epoch = 0;
pConsumer->status = MQ_CONSUMER_STATUS_REBALANCE;
@ -276,20 +287,26 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda
if (updateType == CONSUMER_ADD_REB){
pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
if(pConsumer->rebNewTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
char* topicTmp = taosStrdup(topic);
taosArrayPush(pConsumer->rebNewTopics, &topicTmp);
if (taosArrayPush(pConsumer->rebNewTopics, &topicTmp) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}else if (updateType == CONSUMER_REMOVE_REB) {
pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
if(pConsumer->rebRemovedTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
char* topicTmp = taosStrdup(topic);
taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp);
if (taosArrayPush(pConsumer->rebRemovedTopics, &topicTmp) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
}else if (updateType == CONSUMER_INSERT_SUB){
tstrncpy(pConsumer->clientId, subscribe->clientId, tListLen(pConsumer->clientId));
pConsumer->withTbName = subscribe->withTbName;
@ -297,10 +314,9 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda
pConsumer->autoCommitInterval = subscribe->autoCommitInterval;
pConsumer->resetOffsetCfg = subscribe->resetOffsetCfg;
pConsumer->rebNewTopics = taosArrayDup(subscribe->topicNames, topicNameDup);
if (pConsumer->rebNewTopics == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
pConsumer->assignedTopics = subscribe->topicNames;
@ -310,11 +326,12 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup, int8_t upda
subscribe->topicNames = NULL;
}
return pConsumer;
*ppConsumer = pConsumer;
return 0;
END:
tDeleteSMqConsumerObj(pConsumer);
return NULL;
return code;
}
void tClearSMqConsumerObj(SMqConsumerObj *pConsumer) {
@ -530,27 +547,36 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s
return (void *)buf;
}
SMqSubscribeObj *tNewSubscribeObj(const char *key) {
int32_t tNewSubscribeObj(const char *key, SMqSubscribeObj **ppSub) {
int32_t code = 0;
SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj));
if (pSubObj == NULL) {
return NULL;
}
MND_TMQ_NULL_CHECK(pSubObj);
memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN);
(void)memcpy(pSubObj->key, key, TSDB_SUBSCRIBE_KEY_LEN);
taosInitRWLatch(&pSubObj->lock);
pSubObj->vgNum = 0;
pSubObj->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
// TODO set hash free fp
/*taosHashSetFreeFp(pSubObj->consumerHash, tDeleteSMqConsumerEp);*/
MND_TMQ_NULL_CHECK(pSubObj->consumerHash);
pSubObj->unassignedVgs = taosArrayInit(0, POINTER_BYTES);
return pSubObj;
MND_TMQ_NULL_CHECK(pSubObj->unassignedVgs);
if (ppSub){
*ppSub = pSubObj;
}
return code;
END:
taosMemoryFree(pSubObj);
return code;
}
SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
int32_t tCloneSubscribeObj(const SMqSubscribeObj *pSub, SMqSubscribeObj **ppSub) {
int32_t code = 0;
SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj));
if (pSubNew == NULL) return NULL;
memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
if (pSubNew == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
(void)memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
taosInitRWLatch(&pSubNew->lock);
pSubNew->dbUid = pSub->dbUid;
@ -560,8 +586,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
// TODO set hash free fp
/*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
void *pIter = NULL;
SMqConsumerEp *pConsumerEp = NULL;
while (1) {
@ -576,9 +601,13 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
}
pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp);
pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL);
memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN);
(void)memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN);
pSubNew->qmsg = taosStrdup(pSub->qmsg);
return pSubNew;
if (ppSub) {
*ppSub = pSubNew;
}
END:
return code;
}
void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {

223
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -165,15 +165,24 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create dnode:%s on first deploy", pTrans->id, dnodeObj.ep);
pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
@ -563,10 +572,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
SStatisReq statisReq = {0};
int32_t code = -1;
if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return code;
}
TAOS_CHECK_RETURN(tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq));
if (tsMonitorLogProtocol) {
mInfo("process statis req,\n %s", statisReq.pCont);
@ -586,23 +592,28 @@ static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-dnode-obj");
if (pTrans == NULL) {
code = terrno;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _exit;
}
pDnode->updateTime = taosGetTimestampMs();
SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _exit;
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, tstrerror(code));
code = terrno;
goto _exit;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
code = terrno;
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code));
goto _exit;
}
@ -617,10 +628,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
int32_t code = -1;
if (tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSStatusReq(pReq->pCont, pReq->contLen, &statusReq), NULL, _OVER);
int64_t clusterid = mndGetClusterId(pMnode);
if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) {
@ -634,6 +642,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp);
if (pDnode == NULL) {
mInfo("dnode:%s, not created yet", statusReq.dnodeEp);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
} else {
@ -871,16 +881,25 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", pCreate->fqdn, pCreate->port);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
@ -901,7 +920,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet));
if (NULL == rsp.dnodeList) {
mError("failed to alloc epSet while process dnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -922,7 +941,7 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -935,12 +954,12 @@ static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
_OVER:
if (code != 0) {
mError("failed to get dnode list since %s", terrstr());
mError("failed to get dnode list since %s", tstrerror(code));
}
tFreeSDnodeListRsp(&rsp);
return code;
TAOS_RETURN(code);
}
static void getSlowLogScopeString(int32_t scope, char* result){
@ -980,7 +999,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
rsp.variables = taosArrayInit(16, sizeof(SVariablesInfo));
if (NULL == rsp.variables) {
mError("failed to alloc SVariablesInfo array while process show variables req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -1036,7 +1055,7 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
int32_t rspLen = tSerializeSShowVariablesRsp(NULL, 0, &rsp);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -1049,11 +1068,11 @@ static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) {
_OVER:
if (code != 0) {
mError("failed to get show variables info since %s", terrstr());
mError("failed to get show variables info since %s", tstrerror(code));
}
tFreeSShowVariablesRsp(&rsp);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
@ -1062,23 +1081,17 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
SCreateDnodeReq createReq = {0};
if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0 || (terrno = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) {
code = terrno;
if ((code = grantCheck(TSDB_GRANT_DNODE)) != 0 || (code = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) {
goto _OVER;
}
if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE), NULL, _OVER);
if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) {
terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
code = TSDB_CODE_MND_INVALID_DNODE_EP;
goto _OVER;
}
@ -1086,7 +1099,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode != NULL) {
terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
code = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
goto _OVER;
}
@ -1103,12 +1116,12 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, terrstr());
mError("dnode:%s:%d, failed to create since %s", createReq.fqdn, createReq.port, tstrerror(code));
}
mndReleaseDnode(pMnode, pDnode);
tFreeSCreateDnodeReq(&createReq);
return code;
TAOS_RETURN(code);
}
extern int32_t mndProcessRestoreDnodeReqImpl(SRpcMsg *pReq);
@ -1126,44 +1139,56 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
STrans *pTrans = NULL;
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "drop-dnode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop dnode:%d, force:%d", pTrans->id, pDnode->id, force);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL) goto _OVER;
if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING);
pRaw = NULL;
pRaw = mndDnodeActionEncode(pDnode);
if (pRaw == NULL) goto _OVER;
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
if (pRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
pRaw = NULL;
if (pMObj != NULL) {
mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj, force), NULL, _OVER);
}
if (pQObj != NULL) {
mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj, force), NULL, _OVER);
}
if (pSObj != NULL) {
mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj, force), NULL, _OVER);
}
if (numOfVnodes > 0) {
mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id, force, unsafe), NULL, _OVER);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
code = 0;
@ -1171,7 +1196,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
_OVER:
mndTransDrop(pTrans);
sdbFreeRaw(pRaw);
return code;
TAOS_RETURN(code);
}
static bool mndIsEmptyDnode(SMnode *pMnode, int32_t dnodeId) {
@ -1209,16 +1234,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
SSnodeObj *pSObj = NULL;
SDropDnodeReq dropReq = {0};
if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("dnode:%d, start to drop, ep:%s:%d, force:%s, unsafe:%s", dropReq.dnodeId, dropReq.fqdn, dropReq.port,
dropReq.force ? "true" : "false", dropReq.unsafe ? "true" : "false");
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE), NULL, _OVER);
bool force = dropReq.force;
if (dropReq.unsafe) {
@ -1232,7 +1252,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode == NULL) {
terrno = err;
code = err;
goto _OVER;
}
}
@ -1242,11 +1262,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
if (pMObj != NULL) {
if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
terrno = TSDB_CODE_MND_TOO_FEW_MNODES;
code = TSDB_CODE_MND_TOO_FEW_MNODES;
goto _OVER;
}
if (pMnode->selfDnodeId == dropReq.dnodeId) {
terrno = TSDB_CODE_MND_CANT_DROP_LEADER;
code = TSDB_CODE_MND_CANT_DROP_LEADER;
goto _OVER;
}
}
@ -1255,16 +1275,16 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs());
if (isonline && force) {
terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
code = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER;
}
bool isEmpty = mndIsEmptyDnode(pMnode, pDnode->id);
if (!isonline && !force && !isEmpty) {
terrno = TSDB_CODE_DNODE_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(),
code = TSDB_CODE_DNODE_OFFLINE;
mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, tstrerror(code),
numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL);
goto _OVER;
}
@ -1279,7 +1299,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
}
mndReleaseDnode(pMnode, pDnode);
@ -1287,11 +1307,11 @@ _OVER:
mndReleaseQnode(pMnode, pQObj);
mndReleaseSnode(pMnode, pSObj);
tFreeSDropDnodeReq(&dropReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
terrno = 0;
int32_t code = 0;
char *p = pMCfgReq->config;
while (*p) {
if (*p == ' ') {
@ -1314,12 +1334,12 @@ static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) {
strcpy(pDCfgReq->value, pMCfgReq->value);
}
return 0;
TAOS_RETURN(code);
_err:
mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
code = TSDB_CODE_INVALID_CFG;
TAOS_RETURN(code);
}
static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) {
@ -1349,23 +1369,21 @@ static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq
}
if (code == -1) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
}
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq));
int8_t updateIpWhiteList = 0;
mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) {
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
TAOS_RETURN(code);
}
SDCfgDnodeReq dcfgReq = {0};
@ -1381,26 +1399,26 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) {
mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]",
cfgReq.dnodeId, flag);
terrno = TSDB_CODE_INVALID_CFG;
code = TSDB_CODE_INVALID_CFG;
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
TAOS_RETURN(code);
}
strcpy(dcfgReq.config, "s3blocksize");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
#endif
} else {
if (mndMCfg2DCfg(&cfgReq, &dcfgReq)) goto _err_out;
TAOS_CHECK_GOTO (mndMCfg2DCfg(&cfgReq, &dcfgReq), NULL, _err_out);
if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) {
mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId);
terrno = TSDB_CODE_INVALID_CFG;
code = TSDB_CODE_INVALID_CFG;
goto _err_out;
}
if (strncasecmp(dcfgReq.config, "enableWhiteList", strlen("enableWhiteList")) == 0) {
updateIpWhiteList = 1;
}
if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out;
TAOS_CHECK_GOTO(cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true), NULL, _err_out);
}
{ // audit
@ -1412,15 +1430,15 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq);
int32_t code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);
code = mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq);
// dont care suss or succ;
if (updateIpWhiteList) mndRefreshUserIpWhiteList(pMnode);
return code;
TAOS_RETURN(code);
_err_out:
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
TAOS_RETURN(code);
}
static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) {
@ -1503,17 +1521,16 @@ _exit:
}
static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
int32_t code = 0;
#ifdef TD_ENTERPRISE
SMnode *pMnode = pReq->info.node;
SMCfgDnodeReq cfgReq = {0};
if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq));
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) {
if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE)) != 0) {
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
TAOS_RETURN(code);
}
const STraceId *trace = &pReq->info.traceId;
SDCfgDnodeReq dcfgReq = {0};
@ -1523,13 +1540,13 @@ static int32_t mndProcessCreateEncryptKeyReq(SRpcMsg *pReq) {
tFreeSMCfgDnodeReq(&cfgReq);
return mndProcessCreateEncryptKeyReqImpl(pReq, cfgReq.dnodeId, &dcfgReq);
} else {
terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
code = TSDB_CODE_PAR_INTERNAL_ERROR;
tFreeSMCfgDnodeReq(&cfgReq);
return -1;
TAOS_RETURN(code);
}
#else
return 0;
TAOS_RETURN(code);
#endif
}
@ -1710,7 +1727,7 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) {
// get int32_t value from 'SMCfgDnodeReq'
static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) {
terrno = 0;
int32_t code = 0;
if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) {
goto _err;
}
@ -1725,12 +1742,12 @@ static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32
*pOutValue = atoi(pMCfgReq->value);
}
return 0;
TAOS_RETURN(code);
_err:
mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config);
terrno = TSDB_CODE_INVALID_CFG;
return -1;
code = TSDB_CODE_INVALID_CFG;
TAOS_RETURN(code);
}
SArray *mndGetAllDnodeFqdns(SMnode *pMnode) {

10
source/dnode/mnode/impl/src/mndDump.c
View File

@ -29,12 +29,14 @@ void reportStartup(const char *name, const char *desc) {}
void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
int32_t code = 0;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
int32_t sendSyncReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
int32_t code = 0;
code = TSDB_CODE_INVALID_PTR;
TAOS_RETURN(code);
}
char *i642str(int64_t val) {

157
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -223,6 +223,7 @@ static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) {
}
static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) {
terrno = 0;
SSdb *pSdb = pMnode->pSdb;
SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName);
if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
@ -240,7 +241,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
int32_t code = -1;
STrans *pTrans = NULL;
if ((terrno = grantCheck(TSDB_GRANT_USER)) < 0) {
if ((code = grantCheck(TSDB_GRANT_USER)) < 0) {
return code;
}
@ -260,7 +261,7 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
func.codeSize = pCreate->codeLen;
func.pCode = taosMemoryMalloc(func.codeSize);
if (func.pCode == NULL || func.pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -270,7 +271,11 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
memcpy(func.pCode, pCreate->pCode, func.codeSize);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name);
SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name);
@ -279,31 +284,61 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre
func.createdTime = oldFunc->createdTime;
SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) goto _OVER;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY), NULL, _OVER);
SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) goto _OVER;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY), NULL, _OVER);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER);
} else {
SSdbRaw *pRedoRaw = mndFuncActionEncode(&func);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING), NULL, _OVER);
SSdbRaw *pUndoRaw = mndFuncActionEncode(&func);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED), NULL, _OVER);
SSdbRaw *pCommitRaw = mndFuncActionEncode(&func);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
TAOS_CHECK_GOTO(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY), NULL, _OVER);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
@ -315,32 +350,48 @@ _OVER:
taosMemoryFree(func.pCode);
taosMemoryFree(func.pComment);
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "drop-func");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name);
SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc);
if (pRedoRaw == NULL) goto _OVER;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendRedolog(pTrans, pRedoRaw), NULL, _OVER);
(void)sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc);
if (pUndoRaw == NULL) goto _OVER;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendUndolog(pTrans, pUndoRaw), NULL, _OVER);
(void)sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc);
if (pCommitRaw == NULL) goto _OVER;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
TAOS_CHECK_GOTO(mndTransAppendCommitlog(pTrans, pCommitRaw), NULL, _OVER);
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
@ -355,18 +406,14 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = NULL;
SCreateFuncReq createReq = {0};
if (tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateFuncReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
#ifdef WINDOWS
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
code = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
#endif
mInfo("func:%s, start to create, size:%d", createReq.name, createReq.codeLen);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC), NULL, _OVER);
pFunc = mndAcquireFunc(pMnode, createReq.name);
if (pFunc != NULL) {
@ -378,7 +425,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
mInfo("func:%s, replace function is set", createReq.name);
code = 0;
} else {
terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST;
code = TSDB_CODE_MND_FUNC_ALREADY_EXIST;
goto _OVER;
}
} else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) {
@ -386,22 +433,22 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
}
if (createReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME;
code = TSDB_CODE_MND_INVALID_FUNC_NAME;
goto _OVER;
}
if (createReq.pCode == NULL) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
code = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
code = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
code = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
goto _OVER;
}
@ -410,12 +457,12 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("func:%s, failed to create since %s", createReq.name, terrstr());
mError("func:%s, failed to create since %s", createReq.name, tstrerror(code));
}
mndReleaseFunc(pMnode, pFunc);
tFreeSCreateFuncReq(&createReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
@ -424,18 +471,13 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = NULL;
SDropFuncReq dropReq = {0};
if (tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSDropFuncReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("func:%s, start to drop", dropReq.name);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC), NULL, _OVER);
if (dropReq.name[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_NAME;
code = TSDB_CODE_MND_INVALID_FUNC_NAME;
goto _OVER;
}
@ -446,7 +488,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
code = 0;
goto _OVER;
} else {
terrno = TSDB_CODE_MND_FUNC_NOT_EXIST;
code = TSDB_CODE_MND_FUNC_NOT_EXIST;
goto _OVER;
}
}
@ -456,11 +498,11 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("func:%s, failed to drop since %s", dropReq.name, terrstr());
mError("func:%s, failed to drop since %s", dropReq.name, tstrerror(code));
}
mndReleaseFunc(pMnode, pFunc);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
@ -470,25 +512,25 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
SRetrieveFuncRsp retrieveRsp = {0};
if (tDeserializeSRetrieveFuncReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto RETRIEVE_FUNC_OVER;
}
if (retrieveReq.numOfFuncs <= 0 || retrieveReq.numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE;
code = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE;
goto RETRIEVE_FUNC_OVER;
}
retrieveRsp.numOfFuncs = retrieveReq.numOfFuncs;
retrieveRsp.pFuncInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncInfo));
if (retrieveRsp.pFuncInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER;
}
retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo));
if (retrieveRsp.pFuncExtraInfos == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER;
}
@ -497,6 +539,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName);
if (pFunc == NULL) {
if (terrno != 0) code = terrno;
goto RETRIEVE_FUNC_OVER;
}
@ -541,7 +584,7 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) {
int32_t contLen = tSerializeSRetrieveFuncRsp(NULL, 0, &retrieveRsp);
void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto RETRIEVE_FUNC_OVER;
}
@ -556,7 +599,7 @@ RETRIEVE_FUNC_OVER:
tFreeSRetrieveFuncReq(&retrieveReq);
tFreeSRetrieveFuncRsp(&retrieveRsp);
return code;
TAOS_RETURN(code);
}
static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int32_t len) {

267
source/dnode/mnode/impl/src/mndIndex.c
View File

@ -93,6 +93,7 @@ static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName,
}
int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
@ -110,18 +111,20 @@ int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStb
if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pReq;
action.contLen = contLen;
action.msgType = TDMT_VND_CREATE_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
TAOS_RETURN(code);
}
sdbRelease(pSdb, pVgroup);
}
@ -164,6 +167,7 @@ _err:
return NULL;
}
int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
@ -182,6 +186,8 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
if (pReq == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
STransAction action = {0};
@ -189,7 +195,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
action.pCont = pReq;
action.contLen = len;
action.msgType = TDMT_VND_DROP_INDEX;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
@ -198,7 +204,7 @@ int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbOb
sdbRelease(pSdb, pVgroup);
}
return 0;
TAOS_RETURN(code);
}
void mndCleanupIdx(SMnode *pMnode) {
@ -332,61 +338,91 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
}
int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
return -1;
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) {
sdbFreeRaw(pRedoRaw);
return -1;
}
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
sdbFreeRaw(pCommitRaw);
return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
sdbFreeRaw(pCommitRaw);
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pVgRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) {
int32_t code = 0;
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pVgRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pVgRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pVgRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
// static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
@ -427,10 +463,7 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
SDbObj *pDb = NULL;
SCreateTagIndexReq createReq = {0};
if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("idx:%s start to create", createReq.idxName);
// if (mndCheckCreateIdxReq(&createReq) != 0) {
@ -439,29 +472,30 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByStb(pMnode, createReq.stbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER;
}
pStb = mndAcquireStb(pMnode, createReq.stbName);
if (pStb == NULL) {
mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName);
code = TSDB_CODE_MND_DB_NOT_EXIST;
if (terrno != 0) code = terrno;
goto _OVER;
}
SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx) == 0) {
if ((code = mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx;
} else {
goto _OVER;
}
if (pIdx != NULL) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb);
if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) {
@ -472,32 +506,42 @@ static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stb:%s, failed to create since %s", createReq.idxName, terrstr());
mError("stb:%s, failed to create since %s", createReq.idxName, tstrerror(code));
}
mndReleaseStb(pMnode, pStb);
mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb);
return code;
TAOS_RETURN(code);
}
int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendPrepareLog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
return 0;
TAOS_RETURN(code);
}
static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) {
@ -592,6 +636,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName,
int on) {
int32_t code = 0;
taosRLockLatch(&pOld->lock);
memcpy(pNew, pOld, sizeof(SStbObj));
taosRUnLockLatch(&pOld->lock);
@ -604,28 +649,24 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pOld, tagName, &hasIdx);
if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_NOT_EXIST;
TAOS_RETURN(code);
}
col_id_t colId = pOld->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
return -1;
}
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId));
TAOS_CHECK_RETURN(mndAllocStbSchemas(pOld, pNew));
SSchema *pTag = pNew->pTags + tag;
if (on == 1) {
if (hasIdx && tag != 0) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
} else {
SSCHMEA_SET_IDX_ON(pTag);
}
} else {
if (hasIdx == 0) {
terrno = TSDB_CODE_MND_SMA_NOT_EXIST;
code = TSDB_CODE_MND_SMA_NOT_EXIST;
} else {
SSCHMEA_SET_IDX_OFF(pTag);
pTag->flags = 0;
@ -634,11 +675,15 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
pNew->tagVer++;
SSdbRaw *pCommitRaw = mndStbActionEncode(pNew);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
return 0;
TAOS_RETURN(code);
}
int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) {
// impl later
@ -649,17 +694,17 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
// mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name);
mndTransSetDbName(pTrans, pDb->name, pStb->name);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans);
if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER;
if (mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx), NULL, _OVER);
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
@ -671,7 +716,7 @@ _OVER:
taosMemoryFree(newStb.pCmpr);
}
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
int8_t mndCheckIndexNameByTagName(SMnode *pMnode, SIdxObj *pIdxObj) {
// build index on first tag, and no index name;
@ -725,25 +770,23 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
int8_t hasIdx = 0;
int32_t tag = mndFindSuperTableTagId(pStb, req->colName, &hasIdx);
if (tag < 0) {
terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_NOT_EXIST;
TAOS_RETURN(code);
}
int8_t exist = 0;
if (tag == 0 && hasIdx == 1) {
exist = mndCheckIndexNameByTagName(pMnode, &idxObj);
if (exist) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
}
} else if (hasIdx == 1) {
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
}
col_id_t colId = pStb->pTags[tag].colId;
if (mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId) != 0) {
return -1;
}
TAOS_CHECK_RETURN(mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId));
// SSchema *pTag = pStb->pTags + tag;
// if (IS_IDX_ON(pTag)) {
@ -752,7 +795,7 @@ static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *re
// }
code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj);
return code;
TAOS_RETURN(code);
}
static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) {
@ -763,22 +806,30 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
SStbObj newObj = {0};
pStb = mndAcquireStb(pMnode, pIdx->stb);
if (pStb == NULL) goto _OVER;
if (pStb == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name);
mndTransSetDbName(pTrans, pDb->name, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndTransCheckConflict(pMnode, pTrans), NULL, _OVER);
mndTransSetSerial(pTrans);
if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx), NULL, _OVER);
if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER;
if (mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
@ -789,7 +840,7 @@ _OVER:
mndTransDrop(pTrans);
mndReleaseStb(pMnode, pStb);
return code;
TAOS_RETURN(code);
}
int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
@ -798,13 +849,10 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
SIdxObj *pIdx = NULL;
SDropTagIndexReq req = {0};
if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req), NULL, _OVER);
mInfo("idx:%s, start to drop", req.name);
SSIdx idx = {0};
if (mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx) == 0) {
if ((code = mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx)) == 0) {
pIdx = idx.pIdx;
} else {
goto _OVER;
@ -815,7 +863,7 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
code = 0;
goto _OVER;
} else {
terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST;
code = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST;
goto _OVER;
}
}
@ -823,23 +871,22 @@ int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) {
pDb = mndAcquireDbByIdx(pMnode, req.name);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
if (terrno != 0) code = terrno;
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb), NULL, _OVER);
code = mndDropIdx(pMnode, pReq, pDb, pIdx);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("idx:%s, failed to drop since %s", req.name, terrstr());
mError("idx:%s, failed to drop since %s", req.name, tstrerror(code));
}
mndReleaseIdx(pMnode, pIdx);
mndReleaseDb(pMnode, pDb);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
// do nothing
@ -847,6 +894,7 @@ static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) {
}
int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
@ -856,17 +904,17 @@ int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
if (pIter == NULL) break;
if (pIdx->stbUid == pStb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) {
if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pIdx);
return -1;
TAOS_RETURN(code);
}
}
sdbRelease(pSdb, pIdx);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) {
@ -891,6 +939,7 @@ int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxOb
return -1;
}
int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
@ -900,15 +949,15 @@ int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
if (pIter == NULL) break;
if (pIdx->dbUid == pDb->uid) {
if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) {
if ((code = mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx)) != 0) {
sdbRelease(pSdb, pIdx);
sdbCancelFetch(pSdb, pIter);
return -1;
TAOS_RETURN(code);
}
}
sdbRelease(pSdb, pIdx);
}
return 0;
TAOS_RETURN(code);
}

12
source/dnode/mnode/impl/src/mndIndexCom.c
View File

@ -30,7 +30,7 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA);
SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX);
terrno = 0;
int32_t code = 0;
if (pSma == NULL && pIdx == NULL) return 0;
@ -40,8 +40,8 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pSma;
} else {
mndReleaseSma(pMnode, pSma);
terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_SMA_ALREADY_EXIST;
TAOS_RETURN(code);
}
} else {
if (type == SDB_IDX) {
@ -49,9 +49,9 @@ int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) {
idx->pIdx = pIdx;
} else {
mndReleaseIdx(pMnode, pIdx);
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
return -1;
code = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
TAOS_RETURN(code);
}
}
return 0;
TAOS_RETURN(code);
}

55
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -18,10 +18,11 @@
#include "systable.h"
static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
int32_t code = 0;
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
for (int32_t i = 0; i < colNum; ++i) {
@ -35,10 +36,11 @@ static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t co
}
*pDst = schema;
return 0;
TAOS_RETURN(code);
}
static int32_t mndInsInitMeta(SHashObj *hash) {
int32_t code = 0;
STableMetaRsp meta = {0};
tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName));
@ -55,24 +57,23 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
meta.numOfColumns = pInfosTableMeta[i].colNum;
meta.sysInfo = pInfosTableMeta[i].sysInfo;
if (mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)) {
return -1;
}
TAOS_CHECK_RETURN(mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas));
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
}
return 0;
TAOS_RETURN(code);
}
int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo,
STableMetaRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *pMeta = NULL;
@ -84,40 +85,41 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
if (!sysinfo && pMeta->sysInfo) {
mError("no permission to get schema of table name:%s", tbName);
terrno = TSDB_CODE_PAR_PERMISSION_DENIED;
return -1;
code = TSDB_CODE_PAR_PERMISSION_DENIED;
TAOS_RETURN(code);
}
*pRsp = *pMeta;
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
return 0;
TAOS_RETURN(code);
}
int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) {
int32_t code = 0;
if (NULL == pMnode->infosMeta) {
terrno = TSDB_CODE_APP_ERROR;
return -1;
code = TSDB_CODE_APP_ERROR;
TAOS_RETURN(code);
}
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
TAOS_RETURN(code);
}
strcpy(pRsp->tbName, pMeta->tbName);
@ -129,22 +131,21 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
pRsp->pSchemas = NULL;
return -1;
TAOS_RETURN(code);
}
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt));
return 0;
TAOS_RETURN(code);
}
int32_t mndInitInfos(SMnode *pMnode) {
pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pMnode->infosMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
return mndInsInitMeta(pMnode->infosMeta);

193
source/dnode/mnode/impl/src/mndMain.c
View File

@ -50,8 +50,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
terrno = TSDB_CODE_APP_IS_STOPPING;
code = -1;
code = TSDB_CODE_APP_IS_STOPPING;
} else if (!mndIsLeader(pMnode)) {
code = -1;
} else {
@ -63,7 +62,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) {
#endif
}
taosThreadRwlockUnlock(&pMnode->lock);
return code;
TAOS_RETURN(code);
}
static inline void mndReleaseRpc(SMnode *pMnode) {
@ -78,6 +77,7 @@ static inline void mndReleaseRpc(SMnode *pMnode) {
}
static void *mndBuildTimerMsg(int32_t *pContLen) {
terrno = 0;
SMTimerReq timerReq = {0};
int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq);
@ -329,6 +329,7 @@ static int32_t minCronTime() {
return min <= 1 ? 2 : min;
}
void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
int32_t code = 0;
if (sec % tsTtlPushIntervalSec == 0) {
mndPullupTtl(pMnode);
}
@ -378,14 +379,14 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
}
if (sec % (tsArbHeartBeatIntervalSec) == 0) {
if (mndPullupArbHeartbeat(pMnode) != 0) {
mError("failed to pullup arb heartbeat, since:%s", terrstr());
if ((code = mndPullupArbHeartbeat(pMnode)) != 0) {
mError("failed to pullup arb heartbeat, since:%s", tstrerror(code));
}
}
if (sec % (tsArbCheckSyncIntervalSec) == 0) {
if (mndPullupArbCheckSync(pMnode) != 0) {
mError("failed to pullup arb check sync, since:%s", terrstr());
if ((code = mndPullupArbCheckSync(pMnode)) != 0) {
mError("failed to pullup arb check sync, since:%s", tstrerror(code));
}
}
}
@ -426,17 +427,18 @@ static void *mndThreadFp(void *param) {
}
static int32_t mndInitTimer(SMnode *pMnode) {
int32_t code = 0;
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode) != 0) {
if ((code = taosThreadCreate(&pMnode->thread, &thAttr, mndThreadFp, pMnode)) != 0) {
mError("failed to create timer thread since %s", strerror(errno));
return -1;
TAOS_RETURN(code);
}
taosThreadAttrDestroy(&thAttr);
tmsgReportStartup("mnode-timer", "initialized");
return 0;
TAOS_RETURN(code);
}
static void mndCleanupTimer(SMnode *pMnode) {
@ -447,21 +449,23 @@ static void mndCleanupTimer(SMnode *pMnode) {
}
static int32_t mndCreateDir(SMnode *pMnode, const char *path) {
int32_t code = 0;
pMnode->path = taosStrdup(path);
if (pMnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
if (taosMkDir(pMnode->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
code = terrno;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndInitWal(SMnode *pMnode) {
int32_t code = 0;
char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = {.vgId = 1,
@ -478,9 +482,10 @@ static int32_t mndInitWal(SMnode *pMnode) {
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_MNODE_WAL) == DND_CS_MNODE_WAL){
cfg.encryptAlgorithm = (tsiEncryptScope & DND_CS_MNODE_WAL)? tsiEncryptAlgorithm : 0;
if(tsEncryptKey[0] == '\0'){
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
} else {
code = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
TAOS_RETURN(code);
}
else{
strncpy(cfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
@ -488,11 +493,13 @@ static int32_t mndInitWal(SMnode *pMnode) {
pMnode->pWal = walOpen(path, &cfg);
if (pMnode->pWal == NULL) {
mError("failed to open wal since %s. wal:%s", terrstr(), path);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("failed to open wal since %s. wal:%s", tstrerror(code), path);
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static void mndCloseWal(SMnode *pMnode) {
@ -503,6 +510,7 @@ static void mndCloseWal(SMnode *pMnode) {
}
static int32_t mndInitSdb(SMnode *pMnode) {
int32_t code = 0;
SSdbOpt opt = {0};
opt.path = pMnode->path;
opt.pMnode = pMnode;
@ -510,10 +518,12 @@ static int32_t mndInitSdb(SMnode *pMnode) {
pMnode->pSdb = sdbInit(&opt);
if (pMnode->pSdb == NULL) {
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndOpenSdb(SMnode *pMnode) {
@ -539,48 +549,47 @@ static int32_t mndAllocStep(SMnode *pMnode, char *name, MndInitFp initFp, MndCle
step.initFp = initFp;
step.cleanupFp = cleanupFp;
if (taosArrayPush(pMnode->pSteps, &step) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
return 0;
TAOS_RETURN(0);
}
static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem) != 0) return -1;
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-wal", mndInitWal, mndCloseWal));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndInitSdb, mndCleanupSdb));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-arbgroup", mndInitArbGroup, mndCleanupArbGroup));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact", mndInitCompact, mndCleanupCompact));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-compact-detail", mndInitCompactDetail, mndCleanupCompactDetail));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync));
TAOS_CHECK_RETURN(mndAllocStep(pMnode, "mnode-telem", mndInitTelem, mndCleanupTelem));
return 0;
}
@ -606,17 +615,16 @@ static void mndCleanupSteps(SMnode *pMnode, int32_t pos) {
}
static int32_t mndExecSteps(SMnode *pMnode) {
int32_t code = 0;
int32_t size = taosArrayGetSize(pMnode->pSteps);
for (int32_t pos = 0; pos < size; pos++) {
SMnodeStep *pStep = taosArrayGet(pMnode->pSteps, pos);
if (pStep->initFp == NULL) continue;
if ((*pStep->initFp)(pMnode) != 0) {
int32_t code = terrno;
mError("%s exec failed since %s, start to cleanup", pStep->name, terrstr());
if ((code = (*pStep->initFp)(pMnode)) != 0) {
mError("%s exec failed since %s, start to cleanup", pStep->name, tstrerror(code));
mndCleanupSteps(pMnode, pos);
terrno = code;
return -1;
TAOS_RETURN(code);
} else {
mInfo("%s is initialized", pStep->name);
tmsgReportStartup(pStep->name, "initialized");
@ -624,7 +632,7 @@ static int32_t mndExecSteps(SMnode *pMnode) {
}
pMnode->clusterId = mndGetClusterId(pMnode);
return 0;
TAOS_RETURN(0);
}
static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
@ -639,6 +647,7 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
}
SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
terrno = 0;
mInfo("start to open mnode in %s", path);
SMnode *pMnode = taosMemoryCalloc(1, sizeof(SMnode));
@ -665,7 +674,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
int32_t code = mndCreateDir(pMnode, path);
if (code != 0) {
code = terrno;
mError("failed to open mnode since %s", terrstr());
mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode);
terrno = code;
return NULL;
@ -674,7 +683,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndInitSteps(pMnode);
if (code != 0) {
code = terrno;
mError("failed to open mnode since %s", terrstr());
mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode);
terrno = code;
return NULL;
@ -683,7 +692,7 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
code = mndExecSteps(pMnode);
if (code != 0) {
code = terrno;
mError("failed to open mnode since %s", terrstr());
mError("failed to open mnode since %s", tstrerror(code));
mndClose(pMnode);
terrno = code;
return NULL;
@ -759,45 +768,47 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
if (code != 0) {
mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
terrstr(), code);
tstrerror(code), code);
}
return code;
}
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0;
int32_t code = 0;
if (!IsReq(pMsg)) TAOS_RETURN(code);
if (pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_SCH_MERGE_QUERY ||
pMsg->msgType == TDMT_SCH_QUERY_CONTINUE || pMsg->msgType == TDMT_SCH_QUERY_HEARTBEAT ||
pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK ||
pMsg->msgType == TDMT_SCH_TASK_NOTIFY) {
return 0;
TAOS_RETURN(code);
}
SMnode *pMnode = pMsg->info.node;
taosThreadRwlockRdlock(&pMnode->lock);
if (pMnode->stopped) {
taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_APP_IS_STOPPING;
return -1;
code = TSDB_CODE_APP_IS_STOPPING;
TAOS_RETURN(code);
}
terrno = 0;
SSyncState state = syncGetState(pMnode->syncMgmt.sync);
if (terrno != 0) {
taosThreadRwlockUnlock(&pMnode->lock);
return -1;
code = terrno;
TAOS_RETURN(code);
}
if (state.state != TAOS_SYNC_STATE_LEADER) {
taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_NOT_LEADER;
code = TSDB_CODE_SYN_NOT_LEADER;
goto _OVER;
}
if (!state.restored || !pMnode->restored) {
taosThreadRwlockUnlock(&pMnode->lock);
terrno = TSDB_CODE_SYN_RESTORING;
code = TSDB_CODE_SYN_RESTORING;
goto _OVER;
}
@ -809,7 +820,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
#endif
taosThreadRwlockUnlock(&pMnode->lock);
return 0;
TAOS_RETURN(code);
_OVER:
if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
@ -821,19 +832,17 @@ _OVER:
pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state));
return -1;
TAOS_RETURN(code);
}
const STraceId *trace = &pMsg->info.traceId;
SEpSet epSet = {0};
int32_t tmpCode = terrno;
mndGetMnodeEpSet(pMnode, &epSet);
terrno = tmpCode;
mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "
"role:%s, redirect numOfEps:%d inUse:%d, type:%s",
pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored,
pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code), pMnode->restored, pMnode->stopped, state.restored,
syncStr(state.state), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType));
if (epSet.numOfEps <= 0) return -1;
@ -850,7 +859,7 @@ _OVER:
pMsg->info.rspLen = contLen;
}
return -1;
TAOS_RETURN(code);
}
int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
@ -864,12 +873,12 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
fpExt = pMnode->msgFpExt[TMSG_INDEX(pMsg->msgType)];
if (fpExt == NULL) {
mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
return -1;
code = TSDB_CODE_MSG_NOT_PROCESSED;
TAOS_RETURN(code);
}
}
if (mndCheckMnodeState(pMsg) != 0) return -1;
TAOS_CHECK_RETURN(mndCheckMnodeState(pMsg));
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
if (fp)
@ -883,6 +892,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
} else if (code == 0) {
mGTrace("msg:%p, successfully processed", pMsg);
} else {
//TODO removve this wrong set code
if (code == -1) {
code = terrno;
}
@ -890,7 +900,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg, SQueueInfo *pQueueInfo) {
TMSG_INFO(pMsg->msgType));
}
return code;
TAOS_RETURN(code);
}
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
@ -922,7 +932,8 @@ int64_t mndGenerateUid(const char *name, int32_t len) {
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonStbInfo *pStbInfo, SMonGrantInfo *pGrantInfo) {
if (mndAcquireRpc(pMnode) != 0) return -1;
int32_t code = 0;
TAOS_CHECK_RETURN(mndAcquireRpc(pMnode));
SSdb *pSdb = pMnode->pSdb;
int64_t ms = taosGetTimestampMs();
@ -934,7 +945,9 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pClusterInfo->dnodes == NULL || pClusterInfo->mnodes == NULL || pVgroupInfo->vgroups == NULL ||
pStbInfo->stbs == NULL) {
mndReleaseRpc(pMnode);
return -1;
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
// cluster info
@ -1060,7 +1073,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
}
mndReleaseRpc(pMnode);
return 0;
TAOS_RETURN(code);
}
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {

42
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -771,21 +771,22 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) {
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
return TSDB_CODE_QRY_INVALID_INPUT;
}
} else if (pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL) {
SNode* pAst = NULL;
if ((code = nodesStringToNode(pTopic->ast, &pAst)) != 0) {
code = nodesStringToNode(pTopic->ast, &pAst);
if (code != 0) {
mError("topic:%s, failed to create since %s", pTopic->name, terrstr());
TAOS_RETURN(code);
return code;
}
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
if ((code = qCreateQueryPlan(&cxt, &pPlan, NULL)) != 0) {
code = qCreateQueryPlan(&cxt, &pPlan, NULL);
if (code != 0) {
mError("failed to create topic:%s since %s", pTopic->name, terrstr());
nodesDestroyNode(pAst);
TAOS_RETURN(code);
return code;
}
nodesDestroyNode(pAst);
}
@ -793,18 +794,19 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
if (pPlan) {
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) {
qDestroyQueryPlan(pPlan);
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code);
goto END;
}
SNodeListNode* pNodeListNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
if (pNodeListNode == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
int32_t opNum = LIST_LENGTH(pNodeListNode->pNodeList);
if (opNum != 1) {
qDestroyQueryPlan(pPlan);
code = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
TAOS_RETURN(code);
goto END;
}
pSubplan = (SSubplan*)nodesListGetNode(pNodeListNode->pNodeList, 0);
@ -825,12 +827,18 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
pSub->vgNum++;
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
if (pVgEp == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = pVgroup->vgId;
taosArrayPush(pSub->unassignedVgs, &pVgEp);
if (taosArrayPush(pSub->unassignedVgs, &pVgEp) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pVgEp);
goto END;
}
mInfo("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
sdbRelease(pSdb, pVgroup);
}
@ -838,14 +846,14 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t msgLen;
if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
qDestroyQueryPlan(pPlan);
code = TSDB_CODE_QRY_INVALID_INPUT;
TAOS_RETURN(code);
goto END;
}
} else {
pSub->qmsg = taosStrdup("");
}
END:
qDestroyQueryPlan(pPlan);
TAOS_RETURN(code);
return code;
}

59
source/dnode/mnode/impl/src/mndShow.c
View File

@ -31,17 +31,18 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow);
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) {
int32_t code = 0;
SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr());
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", tstrerror(code));
TAOS_RETURN(code);
}
mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveSysTableReq);
return 0;
TAOS_RETURN(code);
}
void mndCleanupShow(SMnode *pMnode) {
@ -212,6 +213,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
}
static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SShowMgmt *pMgmt = &pMnode->showMgmt;
SShowObj *pShow = NULL;
@ -220,10 +222,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t rowsRead = 0;
mDebug("mndProcessRetrieveSysTableReq start");
SRetrieveTableReq retrieveReq = {0};
if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
TAOS_CHECK_RETURN(tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq));
mDebug("process to retrieve systable req db:%s, tb:%s", retrieveReq.db, retrieveReq.tb);
@ -232,17 +231,17 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (pMeta == NULL) {
pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1;
code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
TAOS_RETURN(code);
}
}
pShow = mndCreateShowObj(pMnode, &retrieveReq);
if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", terrstr());
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta req since %s", tstrerror(code));
TAOS_RETURN(code);
}
pShow->pMeta = pMeta;
@ -250,9 +249,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else {
pShow = mndAcquireShowObj(pMnode, retrieveReq.showId);
if (pShow == NULL) {
terrno = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1;
code = TSDB_CODE_MND_INVALID_SHOWOBJ;
mError("failed to process show-retrieve req:%p since %s", pShow, tstrerror(code));
TAOS_RETURN(code);
}
}
@ -264,9 +263,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr());
return -1;
code = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
TAOS_RETURN(code);
}
mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type);
@ -275,14 +274,16 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else {
memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
}
if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
return -1;
code = -1;
if (retrieveReq.db[0] &&
(code = mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db)) != 0) {
TAOS_RETURN(code);
}
if (pShow->type == TSDB_MGMT_TABLE_USER_FULL) {
if(strcmp(pReq->info.conn.user, "root") != 0){
mError("The operation is not permitted, user:%s, pShow->type:%d", pReq->info.conn.user, pShow->type);
terrno = TSDB_CODE_MND_NO_RIGHTS;
return -1;
code = TSDB_CODE_MND_NO_RIGHTS;
TAOS_RETURN(code);
}
}
@ -308,11 +309,11 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
} else {
rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead);
if (rowsRead < 0) {
terrno = rowsRead;
code = rowsRead;
mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true);
blockDataDestroy(pBlock);
return -1;
TAOS_RETURN(code);
}
pBlock->info.rows = rowsRead;
@ -325,10 +326,10 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
if (pRsp == NULL) {
mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr());
code = TSDB_CODE_OUT_OF_MEMORY;
mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, tstrerror(code));
blockDataDestroy(pBlock);
return -1;
TAOS_RETURN(code);
}
pRsp->handle = htobe64(pShow->id);

701
source/dnode/mnode/impl/src/mndSma.c
View File

File diff suppressed because it is too large Load Diff

172
source/dnode/mnode/impl/src/mndSnode.c
View File

@ -170,38 +170,54 @@ static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew
}
static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING));
TAOS_RETURN(code);
}
static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pUndoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendUndolog(pTrans, pUndoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
}
static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDCreateSnodeReq createReq = {0};
createReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq);
@ -212,23 +228,24 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_CREATE_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -239,12 +256,12 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S
action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendUndoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) {
@ -256,23 +273,27 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
snodeObj.updateTime = snodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto _OVER;
if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto _OVER;
if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) goto _OVER;
if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER;
if (mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetCreateSnodeRedoLogs(pTrans, &snodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateSnodeUndoLogs(pTrans, &snodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateSnodeCommitLogs(pTrans, &snodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
@ -282,15 +303,10 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
SDnodeObj *pDnode = NULL;
SMCreateSnodeReq createReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq), NULL, _OVER);
mInfo("snode:%d, start to create", createReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE), NULL, _OVER);
// pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
// if (pObj != NULL) {
@ -301,13 +317,13 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
// }
if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){
terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
code = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
goto _OVER;
}
pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
if (pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
code = TSDB_CODE_MND_DNODE_NOT_EXIST;
goto _OVER;
}
@ -316,41 +332,52 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr());
return -1;
mError("snode:%d, failed to create since %s", createReq.dnodeId, tstrerror(code));
TAOS_RETURN(code);
}
// mndReleaseSnode(pMnode, pObj);
mndReleaseDnode(pMnode, pDnode);
tFreeSMCreateQnodeReq(&createReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
if (pRedoRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendRedolog(pTrans, pRedoRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING));
TAOS_RETURN(code);
}
static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
}
static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) {
int32_t code = 0;
SDDropSnodeReq dropReq = {0};
dropReq.dnodeId = pDnode->id;
int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq);
void *pReq = taosMemoryMalloc(contLen);
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(code);
}
tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq);
@ -361,20 +388,20 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
action.msgType = TDMT_DND_DROP_SNODE;
action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq);
return -1;
TAOS_RETURN(code);
}
return 0;
TAOS_RETURN(code);
}
int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj, bool force) {
if (pObj == NULL) return 0;
if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1;
if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropSnodeRedoLogs(pTrans, pObj));
TAOS_CHECK_RETURN(mndSetDropSnodeCommitLogs(pTrans, pObj));
if (!force) {
if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
TAOS_CHECK_RETURN(mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj));
}
return 0;
}
@ -383,18 +410,22 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
if (pTrans == NULL) goto _OVER;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
mndTransSetSerial(pTrans);
mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
TAOS_CHECK_GOTO(mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false), NULL, _OVER);
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _OVER);
code = 0;
_OVER:
mndTransDrop(pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
@ -403,23 +434,20 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
SSnodeObj *pObj = NULL;
SMDropSnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
TAOS_CHECK_GOTO(tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq), NULL, _OVER);
mInfo("snode:%d, start to drop", dropReq.dnodeId);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) {
goto _OVER;
}
TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE), NULL, _OVER);
if (dropReq.dnodeId <= 0) {
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
pObj = mndAcquireSnode(pMnode, dropReq.dnodeId);
if (pObj == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OVER;
}
@ -429,12 +457,12 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
mError("snode:%d, failed to drop since %s", dropReq.dnodeId, tstrerror(code));
}
mndReleaseSnode(pMnode, pObj);
tFreeSMCreateQnodeReq(&dropReq);
return code;
TAOS_RETURN(code);
}
static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

942
source/dnode/mnode/impl/src/mndStb.c
View File

File diff suppressed because it is too large Load Diff

1067
source/dnode/mnode/impl/src/mndStream.c
View File

File diff suppressed because it is too large Load Diff

162
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -57,19 +57,30 @@ void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) {
}
}
taosArrayPush(pList, pInfo);
void* p = taosArrayPush(pList, pInfo);
if (p == NULL) {
mError("failed to push failed checkpoint info checkpointId:%" PRId64 " in list", pInfo->checkpointId);
}
}
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
" reset from failed checkpoint");
if (pTrans == NULL) {
STrans *pTrans = NULL;
int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME,
" reset from failed checkpoint", &pTrans);
if (pTrans == NULL || code) {
sdbRelease(pMnode->pSdb, pStream);
return terrno;
}
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code != 0) {
code = mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
@ -79,14 +90,15 @@ int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
return code;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
code = mndTransPrepare(pMnode, pTrans);
if (code != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
return code;
}
sdbRelease(pMnode->pSdb, pStream);
@ -99,8 +111,9 @@ int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t t
int32_t code = TSDB_CODE_SUCCESS;
mndKillTransImpl(pMnode, transId, "");
SStreamObj *pStream = mndGetStreamObj(pMnode, streamId);
if (pStream == NULL) {
SStreamObj *pStream = NULL;
code = mndGetStreamObj(pMnode, streamId, &pStream);
if (pStream == NULL || code != 0) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid);
} else {
@ -159,40 +172,47 @@ int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) {
}
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream");
if (pTrans == NULL) {
STrans *pTrans = NULL;
int32_t code =
doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code != 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr());
return -1;
return code;
}
int32_t code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
if (code) {
return code;
}
// drop all tasks
if (mndStreamSetDropActionFromList(pMnode, pTrans, pList) < 0) {
if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, pList)) < 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr());
mndTransDrop(pTrans);
return -1;
return code;
}
// drop stream
if (mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED) < 0) {
if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) {
mndTransDrop(pTrans);
return -1;
return code;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
return code;
}
mndTransDrop(pTrans);
return 0;
return code;
}
int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
int32_t code = 0;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
@ -204,7 +224,17 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause);
void *pHead = rpcMallocCont(contLen);
tSerializeSMPauseStreamReq(pHead, contLen, &reqPause);
if (pHead == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
sdbRelease(pSdb, pStream);
continue;
}
code = tSerializeSMPauseStreamReq(pHead, contLen, &reqPause);
if (code) {
sdbRelease(pSdb, pStream);
continue;
}
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_PAUSE_STREAM,
@ -213,14 +243,14 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) {
.info = *info,
};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired", pStream->name, reqPause.name,
pStream->uid);
code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired, code:%s", pStream->name, reqPause.name,
pStream->uid, tstrerror(code));
}
sdbRelease(pSdb, pStream);
}
return 0;
return code;
}
int32_t mndProcessStreamHb(SRpcMsg *pReq) {
@ -228,10 +258,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SStreamHbMsg req = {0};
SArray *pFailedChkpt = NULL;
SArray *pOrphanTasks = NULL;
int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if ((code = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if (suspendAllStreams(pMnode, &pReq->info) < 0) {
return -1;
return code;
}
}
@ -241,8 +272,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
if (tDecodeStreamHbMsg(&decoder, &req) < 0) {
tCleanupStreamHbMsg(&req);
tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
code = terrno = TSDB_CODE_INVALID_MSG;
return code;
}
tDecoderClear(&decoder);
@ -251,24 +282,24 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
pFailedChkpt = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
pOrphanTasks = taosArrayInit(4, sizeof(SOrphanTask));
taosThreadMutexLock(&execInfo.lock);
streamMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo);
if (!validateHbMsg(execInfo.pNodeList, req.vgId)) {
mError("vgId:%d not exists in nodeList buf, discarded", req.vgId);
terrno = TSDB_CODE_INVALID_MSG;
code = terrno = TSDB_CODE_INVALID_MSG;
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return -1;
return code;
}
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
if (numOfUpdated > 0) {
mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId);
setNodeEpsetExpiredFlag(req.pUpdateNodes);
(void) setNodeEpsetExpiredFlag(req.pUpdateNodes);
}
bool snodeChanged = false;
@ -280,7 +311,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId);
SOrphanTask oTask = {.streamId = p->id.streamId, .taskId = p->id.taskId, .nodeId = p->nodeId};
taosArrayPush(pOrphanTasks, &oTask);
void* px = taosArrayPush(pOrphanTasks, &oTask);
if (px == NULL) {
mError("Failed to put task into list, taskId:0x%" PRIx64, p->id.taskId);
}
continue;
}
@ -293,11 +327,23 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
.startTs = pChkInfo->consensusTs,
};
SStreamObj *pStream = mndGetStreamObj(pMnode, p->id.streamId);
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
SStreamObj *pStream = NULL;
code = mndGetStreamObj(pMnode, p->id.streamId, &pStream);
if (code) {
code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
continue;
}
SCheckpointConsensusInfo *pInfo = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks);
int32_t numOfTasks = mndGetNumOfStreamTasks(pStream);
SCheckpointConsensusInfo *pInfo = NULL;
code = mndGetConsensusInfo(execInfo.pStreamConsensus, p->id.streamId, numOfTasks, &pInfo);
if (code == 0) {
mndAddConsensusTasks(pInfo, &cp);
} else {
mError("failed to get consensus checkpoint-info");
}
mndReleaseStream(pMnode, pStream);
}
@ -318,7 +364,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
addIntoCheckpointList(pFailedChkpt, &info);
// remove failed trans from pChkptStreams
taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId));
code = taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId));
if (code) {
mError("failed to remove stream:0x%"PRIx64" in checkpoint stream list", p->id.streamId);
}
}
}
@ -338,9 +387,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
// kill the checkpoint trans and then set all tasks status to be normal
if (taosArrayGetSize(pFailedChkpt) > 0) {
bool allReady = true;
if (pMnode != NULL) {
SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady);
SArray *p = NULL;
code = mndTakeVgroupSnapshot(pMnode, &allReady, &p);
taosArrayDestroy(p);
if (code) {
mError("failed to get the vgroup snapshot, ignore it and continue");
}
} else {
allReady = false;
}
@ -352,7 +407,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status",
pInfo->checkpointId, pInfo->transId);
mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId);
code = mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId);
if (code) {
mError("failed to create reset task trans, code:%s", tstrerror(code));
}
}
} else {
mInfo("not all vgroups are ready, wait for next HB from stream tasks to reset the task status");
@ -361,20 +419,19 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
// handle the orphan tasks that are invalid but not removed in some vnodes or snode due to some unknown errors.
if (taosArrayGetSize(pOrphanTasks) > 0) {
mndDropOrphanTasks(pMnode, pOrphanTasks);
code = mndDropOrphanTasks(pMnode, pOrphanTasks);
}
if (pMnode != NULL) { // make sure that the unit test case can work
mndStreamStartUpdateCheckpointInfo(pMnode);
}
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
terrno = TSDB_CODE_SUCCESS;
doSendHbMsgRsp(terrno, &pReq->info, req.vgId, req.msgId);
doSendHbMsgRsp(TSDB_CODE_SUCCESS, &pReq->info, req.vgId, req.msgId);
cleanupAfterProcessHbMsg(&req, pFailedChkpt, pOrphanTasks);
return TSDB_CODE_SUCCESS;
return code;
}
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg
@ -382,7 +439,10 @@ void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doC
if (pMsg != NULL) {
int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_UPDATE_CHKPT_EVT, .pCont = pMsg, .contLen = size};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
int32_t code = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
if (code) {
mError("failed to put into write Queue, code:%s", tstrerror(code));
}
}
}

85
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -24,8 +24,7 @@ typedef struct SKeyInfo {
int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId) {
SStreamTransInfo info = {
.transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId};
taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo));
return 0;
return taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo));
}
int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) {
@ -45,7 +44,10 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt)
SKeyInfo info = {.pKey = pKey, .keyLen = keyLen};
mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name,
pEntry->startTime);
taosArrayPush(pList, &info);
void* p = taosArrayPush(pList, &info);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) {
num++;
@ -57,7 +59,11 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt)
int32_t size = taosArrayGetSize(pList);
for (int32_t i = 0; i < size; ++i) {
SKeyInfo *pKey = taosArrayGet(pList, i);
taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen);
int32_t code = taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen);
if (code != 0) {
taosArrayDestroy(pList);
return code;
}
}
mDebug("clear %d finished stream-trans, remained:%d, active checkpoint trans:%d", size,
@ -79,25 +85,28 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt)
// 2. create/drop/reset/update trans are conflict with any other trans.
bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) {
if (lock) {
taosThreadMutexLock(&execInfo.lock);
streamMutexLock(&execInfo.lock);
}
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) {
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
}
return false;
}
mndStreamClearFinishedTrans(pMnode, NULL);
int32_t code = mndStreamClearFinishedTrans(pMnode, NULL);
if (code) {
mError("failed to clear finish trans, code:%s", tstrerror(code));
}
SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry;
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
}
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) {
@ -122,58 +131,65 @@ bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *p
}
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
}
return false;
}
int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) {
taosThreadMutexLock(&execInfo.lock);
streamMutexLock(&execInfo.lock);
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) {
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
return 0;
}
mndStreamClearFinishedTrans(pMnode, NULL);
int32_t code = mndStreamClearFinishedTrans(pMnode, NULL);
if (code) {
mError("failed to clear finish trans, code:%s", tstrerror(code));
}
SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0 ||
strcmp(tInfo.name, MND_STREAM_CHKPT_UPDATE_NAME) == 0) {
return tInfo.transId;
}
} else {
taosThreadMutexUnlock(&execInfo.lock);
streamMutexUnlock(&execInfo.lock);
}
return 0;
}
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (pTrans == NULL) {
int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name,
const char *pMsg, STrans ** pTrans1) {
*pTrans1 = NULL;
terrno = 0;
STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (p == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return terrno;
}
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id);
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, p) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
mndTransDrop(pTrans);
return NULL;
mndTransDrop(p);
return terrno;
}
terrno = 0;
return pTrans;
*pTrans1 = p;
return 0;
}
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@ -228,21 +244,21 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status)
if (pCommitRaw == NULL) {
mError("failed to encode stream since %s", terrstr());
mndTransDrop(pTrans);
return -1;
return terrno;
}
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
sdbFreeRaw(pCommitRaw);
mndTransDrop(pTrans);
return -1;
return terrno;
}
if (sdbSetRawStatus(pCommitRaw, status) != 0) {
mError("stream trans:%d failed to set raw status:%d since %s", pTrans->id, status, terrstr());
sdbFreeRaw(pCommitRaw);
mndTransDrop(pTrans);
return -1;
return terrno;
}
return 0;
@ -272,8 +288,9 @@ int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t len) {
continue;
}
SStreamObj *pStream = mndGetStreamObj(pMnode, pTransInfo->streamId);
if (pStream != NULL) {
SStreamObj *pStream = NULL;
int32_t code = mndGetStreamObj(pMnode, pTransInfo->streamId, &pStream);
if (pStream != NULL && code == 0) {
if (identicalName(pStream->sourceDb, pDBName, len)) {
mndKillTransImpl(pMnode, pTransInfo->transId, pStream->sourceDb);
} else if (identicalName(pStream->targetDb, pDBName, len)) {
@ -299,8 +316,12 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) {
void *pKey = taosHashGetKey(pDb, &len);
char *p = strndup(pKey, len);
int32_t code = doKillCheckpointTrans(pMnode, pKey, len);
if (code) {
mError("failed to kill trans, transId:%p", pKey)
} else {
mDebug("clear checkpoint trans in Db:%s", p);
doKillCheckpointTrans(pMnode, pKey, len);
}
taosMemoryFree(p);
}

515
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -28,20 +28,20 @@ struct SStreamTaskIter {
int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId);
SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) {
SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
if (pIter == NULL) {
int32_t createStreamTaskIter(SStreamObj* pStream, SStreamTaskIter** pIter) {
*pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter));
if (*pIter == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return terrno;
}
pIter->level = -1;
pIter->ordinalIndex = 0;
pIter->pStream = pStream;
pIter->totalLevel = taosArrayGetSize(pStream->tasks);
pIter->pTask = NULL;
(*pIter)->level = -1;
(*pIter)->ordinalIndex = 0;
(*pIter)->pStream = pStream;
(*pIter)->totalLevel = taosArrayGetSize(pStream->tasks);
(*pIter)->pTask = NULL;
return pIter;
return 0;
}
bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
@ -72,19 +72,27 @@ bool streamTaskIterNextTask(SStreamTaskIter* pIter) {
return false;
}
SStreamTask* streamTaskIterGetCurrent(SStreamTaskIter* pIter) {
return pIter->pTask;
int32_t streamTaskIterGetCurrent(SStreamTaskIter* pIter, SStreamTask** pTask) {
if (pTask) {
*pTask = pIter->pTask;
if (*pTask != NULL) {
return TSDB_CODE_SUCCESS;
}
}
return TSDB_CODE_INVALID_PARA;
}
void destroyStreamTaskIter(SStreamTaskIter* pIter) {
taosMemoryFree(pIter);
}
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
int32_t mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady, SArray** pList) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SVgObj *pVgroup = NULL;
int32_t replica = -1; // do the replica check
int32_t code = 0;
*allReady = true;
SArray *pVgroupList = taosArrayInit(4, sizeof(SNodeEntry));
@ -131,10 +139,15 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
}
char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf));
(void) epsetToStr(&entry.epset, buf, tListLen(buf));
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
taosArrayPush(pVgroupList, &entry);
}
sdbRelease(pSdb, pVgroup);
}
@ -145,43 +158,57 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
break;
}
SNodeEntry entry = {0};
addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port);
entry.nodeId = SNODE_HANDLE;
SNodeEntry entry = {.nodeId = SNODE_HANDLE};
code = addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port);
if (code) {
sdbRelease(pSdb, pObj);
continue;
}
char buf[256] = {0};
epsetToStr(&entry.epset, buf, tListLen(buf));
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
(void) epsetToStr(&entry.epset, buf, tListLen(buf));
void* p = taosArrayPush(pVgroupList, &entry);
if (p == NULL) {
mError("failed to put entry in vgroup list, nodeId:%d code:out of memory", entry.nodeId);
} else {
mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf);
}
taosArrayPush(pVgroupList, &entry);
sdbRelease(pSdb, pObj);
}
return pVgroupList;
*pList = pVgroupList;
return code;
}
SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId) {
int32_t mndGetStreamObj(SMnode *pMnode, int64_t streamId, SStreamObj **pStream) {
void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
*pStream = NULL;
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) {
if (pStream->uid == streamId) {
SStreamObj *p = NULL;
while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&p)) != NULL) {
if (p->uid == streamId) {
sdbCancelFetch(pSdb, pIter);
return pStream;
*pStream = p;
return TSDB_CODE_SUCCESS;
}
sdbRelease(pSdb, pStream);
sdbRelease(pSdb, p);
}
return NULL;
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
}
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName) {
STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans != NULL) {
mInfo("kill active transId:%d in Db:%s", transId, pDbName);
mndKillTrans(pMnode, pTrans);
int32_t code = mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans);
if (code) {
mError("failed to kill trans:%d", pTrans->id);
}
} else {
mError("failed to acquire trans in Db:%s, transId:%d", pDbName, transId);
}
@ -197,11 +224,16 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj);
if (pIter != NULL) {
addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
int32_t code = addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
sdbRelease(pMnode->pSdb, pObj);
sdbCancelFetch(pMnode->pSdb, pIter);
if (code) {
*hasEpset = false;
mError("failed to set epset");
} else {
*hasEpset = true;
return TSDB_CODE_SUCCESS;
}
return code;
} else {
mError("failed to acquire snode epset");
return TSDB_CODE_INVALID_PARA;
@ -223,12 +255,14 @@ int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t
}
static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) {
terrno = 0;
SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq));
if (pReq == NULL) {
mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno;
}
pReq->head.vgId = htonl(pTask->info.nodeId);
@ -242,31 +276,45 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) {
terrno = code;
taosMemoryFree(pReq);
return -1;
return terrno;
}
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
return terrno;
}
mDebug("set the resume action for trans:%d", pTrans->id);
return 0;
}
SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
int32_t mndGetStreamTask(STaskId *pId, SStreamObj *pStream, SStreamTask **pTask) {
*pTask = NULL;
SStreamTask *p = NULL;
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
if (pTask->id.taskId == pId->taskId) {
code = streamTaskIterGetCurrent(pIter, &p);
if (code) {
continue;
}
if (p->id.taskId == pId->taskId) {
destroyStreamTaskIter(pIter);
return pTask;
*pTask = p;
return 0;
}
}
destroyStreamTaskIter(pIter);
return NULL;
return TSDB_CODE_FAILED;
}
int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
@ -280,13 +328,25 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream) {
}
int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
if (doSetResumeAction(pTrans, pMnode, pTask, igUntreated) < 0) {
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code || pTask == NULL) {
destroyStreamTaskIter(pIter);
return -1;
return code;
}
code = doSetResumeAction(pTrans, pMnode, pTask, igUntreated);
if (code) {
destroyStreamTaskIter(pIter);
return code;
}
if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__PAUSE) {
@ -303,7 +363,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno;
}
pReq->head.vgId = htonl(pTask->info.nodeId);
@ -320,25 +380,38 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
}
char buf[256] = {0};
epsetToStr(&epset, buf, tListLen(buf));
(void) epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
return code;
}
return 0;
}
int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
if (doSetPauseAction(pMnode, pTrans, pTask) < 0) {
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
return -1;
return code;
}
code = doSetPauseAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
}
if (atomic_load_8(&pTask->status.taskStatus) != TASK_STATUS__PAUSE) {
@ -348,14 +421,14 @@ int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
}
destroyStreamTaskIter(pIter);
return 0;
return code;
}
static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno;
}
pReq->head.vgId = htonl(pTask->info.nodeId);
@ -366,28 +439,40 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { // no valid epset, return directly without redoAction
terrno = code;
return -1;
return code;
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
return code;
}
return 0;
}
int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
SStreamTaskIter *pIter = NULL;
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
while(streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
if (doSetDropAction(pMnode, pTrans, pTask) < 0) {
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
return -1;
return code;
}
code = doSetDropAction(pMnode, pTrans, pTask);
if (code) {
destroyStreamTaskIter(pIter);
return code;
}
}
destroyStreamTaskIter(pIter);
@ -398,7 +483,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno;
}
pReq->head.vgId = htonl(pTask->nodeId);
@ -409,16 +494,15 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->taskId, pTask->nodeId);
if (code != TSDB_CODE_SUCCESS || (!hasEpset)) { // no valid epset, return directly without redoAction
terrno = code;
taosMemoryFree(pReq);
return -1;
return code;
}
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) {
taosMemoryFree(pReq);
return -1;
return code;
}
return 0;
@ -427,19 +511,35 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray* pList) {
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SOrphanTask* pTask = taosArrayGet(pList, i);
int32_t code = doSetDropActionFromId(pMnode, pTrans, pTask);
if (code != 0) {
return code;
} else {
mDebug("add drop task:0x%x action to drop orphan task", pTask->taskId);
doSetDropActionFromId(pMnode, pTrans, pTask);
}
}
return 0;
}
static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
int32_t transId) {
int32_t code = 0;
pMsg->streamId = pId->streamId;
pMsg->taskId = pId->taskId;
pMsg->transId = transId;
pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
if (pMsg->pNodeList == NULL) {
mError("failed to prepare node list, code:out of memory");
code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code == 0) {
void *p = taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
if (p == NULL) {
mError("failed to add update node list into nodeList");
}
}
}
static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
@ -454,7 +554,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList);
return -1;
return terrno;
}
int32_t tlen = sizeof(SMsgHead) + blen;
@ -463,13 +563,18 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosArrayDestroy(req.pNodeList);
return -1;
return terrno;
}
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen);
tEncodeStreamTaskUpdateMsg(&encoder, &req);
code = tEncodeStreamTaskUpdateMsg(&encoder, &req);
if (code == -1) {
tEncoderClear(&encoder);
taosArrayDestroy(req.pNodeList);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)buf;
pMsgHead->contLen = htonl(tlen);
@ -487,15 +592,20 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
void *pBuf = NULL;
int32_t len = 0;
streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
int32_t code = streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
if (code) {
return code;
}
doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
code = doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
if (code) {
return code;
}
SEpSet epset = {0};
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) {
terrno = code;
return code;
}
@ -510,16 +620,30 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
// build trans to update the epset
int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
taosWLockLatch(&pStream->lock);
SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
int32_t code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return -1;
return code;
}
}
@ -558,16 +682,30 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
}
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock);
SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
int32_t code = doSetResetAction(pMnode, pTrans, pTask);
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
code = doSetResetAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return -1;
return code;
}
}
@ -581,8 +719,12 @@ static void freeTaskList(void* param) {
taosArrayDestroy(*pList);
}
void mndInitExecInfo() {
taosThreadMutexInit(&execInfo.lock, NULL);
int32_t mndInitExecInfo() {
int32_t code = taosThreadMutexInit(&execInfo.lock, NULL);
if (code) {
return code;
}
_hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
@ -596,19 +738,24 @@ void mndInitExecInfo() {
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pStreamConsensus, freeTaskList);
return 0;
}
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
SArray *pValidList = taosArrayInit(4, sizeof(SNodeEntry));
int32_t size = taosArrayGetSize(pNodeSnapshot);
int32_t oldSize = taosArrayGetSize(execInfo.pNodeList);
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) {
for (int32_t i = 0; i < oldSize; ++i) {
SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i);
for (int32_t j = 0; j < size; ++j) {
SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
if (pEntry->nodeId == p->nodeId) {
taosArrayPush(pValidList, p);
void* px = taosArrayPush(pValidList, p);
if (px == NULL) {
mError("failed to put node into list, nodeId:%d", p->nodeId);
}
break;
}
}
@ -617,7 +764,8 @@ void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
taosArrayDestroy(execInfo.pNodeList);
execInfo.pNodeList = pValidList;
mDebug("remain %d valid node entries after clean expired nodes info", (int32_t)taosArrayGetSize(pValidList));
mDebug("remain %d valid node entries after clean expired nodes info, prev size:%d",
(int32_t)taosArrayGetSize(pValidList), oldSize);
}
int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
@ -626,7 +774,10 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
return TSDB_CODE_SUCCESS;
}
taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
int32_t code = taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
if (code) {
return code;
}
for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
@ -645,28 +796,45 @@ int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) {
STaskId *pId = taosArrayGet(pTaskIds, i);
doRemoveTasks(pExecInfo, pId);
int32_t code = doRemoveTasks(pExecInfo, pId);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, pId->taskId);
}
}
}
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexLock(&pExecNode->lock);
SStreamTaskIter *pIter = NULL;
streamMutexLock(&pExecNode->lock);
// 1. remove task entries
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
streamMutexUnlock(&pExecNode->lock);
mError("failed to create stream task iter:%s", pStream->name);
return;
}
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
continue;
}
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
doRemoveTasks(pExecNode, &id);
code = doRemoveTasks(pExecNode, &id);
if (code) {
mError("failed to remove task in buffer list, 0x%"PRIx64, id.taskId);
}
}
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
// 2. remove stream entry in consensus hash table
mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid);
(void) mndClearConsensusCheckpointId(execInfo.pStreamConsensus, pStream->uid);
taosThreadMutexUnlock(&pExecNode->lock);
streamMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter);
}
@ -697,7 +865,10 @@ int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
void* p = taosArrayPush(pRemovedTasks, pId);
if (p == NULL) {
mError("failed to put task entry into remove list, taskId:0x%" PRIx64, pId->taskId);
}
}
}
@ -759,29 +930,43 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas
}
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
SStreamTaskIter *pIter = NULL;
taosWLockLatch(&pStream->lock);
int32_t code = createStreamTaskIter(pStream, &pIter);
if (code) {
taosWUnLockLatch(&pStream->lock);
mError("failed to create stream task iter:%s", pStream->name);
return code;
}
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
SStreamTask *pTask = NULL;
code = streamTaskIterGetCurrent(pIter, &pTask);
if (code) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return code;
}
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return -1;
return code;
}
}
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return 0;
return code;
}
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
int32_t code = 0;
mDebug("start to scan checkpoint report info");
@ -789,10 +974,15 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SArray *pList = *(SArray **)pIter;
STaskChkptInfo *pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId);
if (pStream == NULL) {
SStreamObj *pStream = NULL;
code = mndGetStreamObj(pMnode, pInfo->streamId, &pStream);
if (pStream == NULL || code != 0) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId);
void* p = taosArrayPush(pDropped, &pInfo->streamId);
if (p == NULL) {
mError("failed to put stream into drop list:0x%" PRIx64, pInfo->streamId);
}
continue;
}
@ -805,10 +995,14 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId);
void* p = taosArrayPush(pDropped, &pInfo->streamId);
if (p == NULL) {
mError("failed to remove stream:0x%" PRIx64, pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
}
} else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId);
@ -829,7 +1023,10 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
if (size > 0) {
for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
code = taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
if (code) {
mError("failed to remove stream in buf:0x%"PRIx64, streamId);
}
}
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
@ -854,29 +1051,30 @@ static int32_t mndStreamSetChkptIdAction(SMnode *pMnode, STrans *pTrans, SStream
int32_t blen;
tEncodeSize(tEncodeRestoreCheckpointInfo, &req, blen, code);
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
int32_t tlen = sizeof(SMsgHead) + blen;
void *pBuf = taosMemoryMalloc(tlen);
if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return terrno = TSDB_CODE_OUT_OF_MEMORY;
}
void *abuf = POINTER_SHIFT(pBuf, sizeof(SMsgHead));
SEncoder encoder;
tEncoderInit(&encoder, abuf, tlen);
tEncodeRestoreCheckpointInfo(&encoder, &req);
code = tEncodeRestoreCheckpointInfo(&encoder, &req);
tEncoderClear(&encoder);
if (code == -1) {
taosMemoryFree(pBuf);
return code;
}
SMsgHead *pMsgHead = (SMsgHead *)pBuf;
pMsgHead->contLen = htonl(tlen);
pMsgHead->vgId = htonl(pTask->info.nodeId);
tEncoderClear(&encoder);
SEpSet epset = {0};
bool hasEpset = false;
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
@ -898,17 +1096,28 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
char msg[128] = {0};
snprintf(msg, tListLen(msg), "set consen-chkpt-id for task:0x%x", taskId);
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg);
if (pTrans == NULL) {
STrans *pTrans = NULL;
int32_t code = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_CONSEN_NAME, msg, &pTrans);
if (pTrans == NULL || code != 0) {
return terrno;
}
STaskId id = {.streamId = pStream->uid, .taskId = taskId};
SStreamTask *pTask = mndGetStreamTask(&id, pStream);
ASSERT(pTask);
SStreamTask *pTask = NULL;
code = mndGetStreamTask(&id, pStream, &pTask);
if (code) {
mError("failed to get task:0x%x in stream:%s, failed to create consensus-checkpointId", taskId, pStream->name);
sdbRelease(pMnode->pSdb, pStream);
return code;
}
/*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid);
int32_t code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts);
code = mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_CONSEN_NAME, pStream->uid);
if (code) {
sdbRelease(pMnode->pSdb, pStream);
return code;
}
code = mndStreamSetChkptIdAction(pMnode, pTrans, pTask, checkpointId, ts);
if (code != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
@ -916,17 +1125,18 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
}
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) {
if (code) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
return code;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
code = mndTransPrepare(pMnode, pTrans);
if (code) {
mError("trans:%d, failed to prepare set consensus-chkptId trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
return code;
}
sdbRelease(pMnode->pSdb, pStream);
@ -935,10 +1145,13 @@ int32_t mndCreateSetConsensusChkptIdTrans(SMnode *pMnode, SStreamObj *pStream, i
return TSDB_CODE_ACTION_IN_PROGRESS;
}
SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks) {
void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId));
if (pInfo != NULL) {
return (SCheckpointConsensusInfo*)pInfo;
int32_t mndGetConsensusInfo(SHashObj* pHash, int64_t streamId, int32_t numOfTasks, SCheckpointConsensusInfo **pInfo) {
*pInfo = NULL;
void* px = taosHashGet(pHash, &streamId, sizeof(streamId));
if (px != NULL) {
*pInfo = px;
return 0;
}
SCheckpointConsensusInfo p = {
@ -947,10 +1160,14 @@ SCheckpointConsensusInfo* mndGetConsensusInfo(SHashObj* pHash, int64_t streamId,
.streamId = streamId,
};
taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
int32_t code = taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
if (code == 0) {
void *pChkptInfo = (SCheckpointConsensusInfo *)taosHashGet(pHash, &streamId, sizeof(streamId));
return pChkptInfo;
*pInfo = pChkptInfo;
} else {
*pInfo = NULL;
}
return code;
}
// no matter existed or not, add the request into info list anyway, since we need to send rsp mannually
@ -971,12 +1188,16 @@ void mndAddConsensusTasks(SCheckpointConsensusInfo *pInfo, const SRestoreCheckpo
}
}
taosArrayPush(pInfo->pTaskList, &info);
void *p = taosArrayPush(pInfo->pTaskList, &info);
if (p == NULL) {
mError("s-task:0x%x failed to put task into consensus-checkpointId list, code: out of memory", info.req.taskId);
} else {
int32_t num = taosArrayGetSize(pInfo->pTaskList);
mDebug("s-task:0x%x checkpointId:%" PRId64 " added into consensus-checkpointId list, stream:0x%" PRIx64
" waiting tasks:%d",
pRestoreInfo->taskId, pRestoreInfo->checkpointId, pRestoreInfo->streamId, num);
}
}
void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
taosArrayDestroy(pInfo->pTaskList);
@ -984,22 +1205,18 @@ void mndClearConsensusRspEntry(SCheckpointConsensusInfo* pInfo) {
}
int64_t mndClearConsensusCheckpointId(SHashObj* pHash, int64_t streamId) {
taosHashRemove(pHash, &streamId, sizeof(streamId));
int32_t code = 0;
int32_t numOfStreams = taosHashGetSize(pHash);
mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list after new checkpoint generated, remain:%d", streamId,
numOfStreams);
if (numOfStreams == 0) {
return TSDB_CODE_SUCCESS;
}
//int32_t mndRegisterConsensusChkptId(SHashObj* pHash, int64_t streamId) {
// void* pInfo = taosHashGet(pHash, &streamId, sizeof(streamId));
// ASSERT(pInfo == NULL);
//
// SCheckpointConsensusInfo p = {.genTs = taosGetTimestampMs(), .checkpointId = 0, .pTaskList = NULL};
// taosHashPut(pHash, &streamId, sizeof(streamId), &p, sizeof(p));
//
// SCheckpointConsensusInfo* pChkptInfo = (SCheckpointConsensusInfo*)taosHashGet(pHash, &streamId, sizeof(streamId));
// ASSERT(pChkptInfo->genTs > 0 && pChkptInfo->checkpointId == 0);
// mDebug("s-task:0x%" PRIx64 " set the initial consensus-checkpointId:0", streamId);
// return TSDB_CODE_SUCCESS;
//}
code = taosHashRemove(pHash, &streamId, sizeof(streamId));
if (code == 0) {
mDebug("drop stream:0x%" PRIx64 " in consensus-checkpointId list, remain:%d", streamId, numOfStreams);
} else {
mError("failed to remove stream:0x%"PRIx64" in consensus-checkpointId list, remain:%d", streamId, numOfStreams);
}
return code;
}

767
source/dnode/mnode/impl/src/mndSubscribe.c
View File

File diff suppressed because it is too large Load Diff

51
source/dnode/mnode/impl/src/mndSync.c
View File

@ -123,15 +123,16 @@ _OUT:
}
static int32_t mndTransValidateImp(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (pTrans->stage == TRN_STAGE_PREPARE) {
if (mndTransCheckConflict(pMnode, pTrans) < 0) {
if ((code = mndTransCheckConflict(pMnode, pTrans)) < 0) {
mError("trans:%d, failed to validate trans conflicts.", pTrans->id);
return -1;
TAOS_RETURN(code);
}
return mndTransValidatePrepareStage(pMnode, pTrans);
}
return 0;
TAOS_RETURN(code);
}
static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
@ -139,10 +140,18 @@ static int32_t mndTransValidate(SMnode *pMnode, SSdbRaw *pRaw) {
int32_t code = -1;
SSdbRow *pRow = mndTransDecode(pRaw);
if (pRow == NULL) goto _OUT;
if (pRow == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
pTrans = sdbGetRowObj(pRow);
if (pTrans == NULL) goto _OUT;
if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _OUT;
}
code = mndTransValidateImp(pMnode, pTrans);
@ -150,11 +159,10 @@ _OUT:
if (pTrans) mndTransDropData(pTrans);
if (pRow) taosMemoryFreeClear(pRow);
if (code) terrno = (terrno ? terrno : TSDB_CODE_MND_TRANS_CONFLICT);
return code;
TAOS_RETURN(code);
}
int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
terrno = TSDB_CODE_SUCCESS;
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SSdbRaw *pRaw = pMsg->pCont;
STrans *pTrans = NULL;
@ -163,7 +171,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (transId <= 0) {
mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq);
terrno = TSDB_CODE_INVALID_MSG;
code = TSDB_CODE_INVALID_MSG;
goto _OUT;
}
@ -176,7 +184,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) {
mError("trans:%d, failed to validate requested trans since %s", transId, terrstr());
code = 0;
pMeta->code = terrno;
pMeta->code = code;
goto _OUT;
}
@ -184,13 +192,15 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
if (code != 0) {
mError("trans:%d, failed to write to sdb since %s", transId, terrstr());
code = 0;
pMeta->code = terrno;
pMeta->code = code;
goto _OUT;
}
pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans == NULL) {
mError("trans:%d, not found while execute in mnode since %s", transId, terrstr());
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mError("trans:%d, not found while execute in mnode since %s", transId, tstrerror(code));
goto _OUT;
}
@ -207,7 +217,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) {
_OUT:
if (pTrans) mndReleaseTrans(pMnode, pTrans);
return code;
TAOS_RETURN(code);
}
static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) {
@ -258,7 +268,7 @@ _OUT:
mndPostMgmtCode(pMnode, code ? code : pMeta->code);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return code;
TAOS_RETURN(code);
}
SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) {
@ -466,16 +476,18 @@ int32_t mndInitSync(SMnode *pMnode) {
pNode->clusterId);
}
int32_t code = 0;
tsem_init(&pMgmt->syncSem, 0, 0);
pMgmt->sync = syncOpen(&syncInfo, true);
if (pMgmt->sync <= 0) {
mError("failed to open sync since %s", terrstr());
return -1;
if (terrno != 0) code = terrno;
mError("failed to open sync since %s", tstrerror(code));
TAOS_RETURN(code);
}
pMnode->pSdb->sync = pMgmt->sync;
mInfo("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
return 0;
TAOS_RETURN(code);
}
void mndCleanupSync(SMnode *pMnode) {
@ -518,10 +530,10 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)};
if (req.contLen <= 0) return -1;
if (req.contLen <= 0) return TSDB_CODE_OUT_OF_MEMORY;
req.pCont = rpcMallocCont(req.contLen);
if (req.pCont == NULL) return -1;
if (req.pCont == NULL) return TSDB_CODE_OUT_OF_MEMORY;
memcpy(req.pCont, pRaw, req.contLen);
taosThreadMutexLock(&pMgmt->lock);
@ -531,8 +543,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosThreadMutexUnlock(&pMgmt->lock);
rpcFreeCont(req.pCont);
terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED;
return terrno;
TAOS_RETURN(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED);
}
mInfo("trans:%d, will be proposed", transId);

456
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -44,8 +44,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq);
static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
int32_t mndInitTopic(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_TOPIC,
@ -70,9 +68,16 @@ int32_t mndInitTopic(SMnode *pMnode) {
void mndCleanupTopic(SMnode *pMnode) {}
const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) {
//
return strchr(topic, '.') + 1;
void mndTopicGetShowName(const char* fullTopic, char* topic) {
if (fullTopic == NULL) {
return;
}
char* tmp = strchr(fullTopic, '.');
if (tmp == NULL) {
tstrncpy(topic, fullTopic, TSDB_TOPIC_FNAME_LEN);
}else {
tstrncpy(topic, tmp+1, TSDB_TOPIC_FNAME_LEN);
}
}
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
@ -129,7 +134,9 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
goto TOPIC_ENCODE_OVER;
}
void *aswBuf = swBuf;
taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema);
if(taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema) < 0){
goto TOPIC_ENCODE_OVER;
}
SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER);
}
@ -259,7 +266,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
}
int16_t colId;
SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER);
taosArrayPush(pTopic->ntbColIds, &colId);
(void)taosArrayPush(pTopic->ntbColIds, &colId);
}
SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER);
@ -289,25 +296,25 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
taosMemoryFreeClear(pTopic->ast);
taosMemoryFreeClear(pTopic->physicalPlan);
if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema);
taosArrayDestroy(pTopic->ntbColIds);
(void)taosArrayDestroy(pTopic->ntbColIds);
return 0;
}
static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) {
mTrace("topic:%s perform update action", pOldTopic->name);
atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime);
atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
(void)atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime);
(void)atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
return 0;
}
SMqTopicObj *mndAcquireTopic(SMnode *pMnode, const char *topicName) {
int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic) {
SSdb *pSdb = pMnode->pSdb;
SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
*pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (*pTopic == NULL) {
return TSDB_CODE_MND_TOPIC_NOT_EXIST;
}
return pTopic;
return TDB_CODE_SUCCESS;
}
void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
@ -316,25 +323,23 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
}
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
terrno = TSDB_CODE_MND_INVALID_TOPIC;
if (pCreate->sql == NULL) return -1;
if (pCreate->sql == NULL) return TSDB_CODE_MND_INVALID_TOPIC;
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
if (pCreate->ast == NULL || pCreate->ast[0] == 0) return -1;
if (pCreate->ast == NULL || pCreate->ast[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
if (pCreate->subStbName[0] == 0) return -1;
if (pCreate->subStbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {
if (pCreate->subDbName[0] == 0) return -1;
if (pCreate->subDbName[0] == 0) return TSDB_CODE_MND_INVALID_TOPIC;
}
terrno = TSDB_CODE_SUCCESS;
return 0;
}
static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
SNodeList *pNodeList = NULL;
nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList);
int32_t code = 0;
MND_TMQ_RETURN_CHECK(nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList));
int64_t suid = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->suid;
int8_t tableType = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->tableType;
if (tableType == TSDB_CHILD_TABLE) {
@ -345,17 +350,19 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
SColumnNode *pCol = (SColumnNode *)pNode;
if (pCol->tableType == TSDB_NORMAL_TABLE) {
pTopic->ntbUid = pCol->tableId;
taosArrayPush(pTopic->ntbColIds, &pCol->colId);
MND_TMQ_NULL_CHECK(taosArrayPush(pTopic->ntbColIds, &pCol->colId));
}
}
}
nodesDestroyList(pNodeList);
return 0;
END:
return code;
}
static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj *topicObj){
STqCheckInfo info;
memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN);
STqCheckInfo info = {0};
(void)memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN);
info.ntbUid = topicObj->ntbUid;
info.colIdList = topicObj->ntbColIds;
// broadcast forbid alter info
@ -378,16 +385,17 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj
int32_t len;
tEncodeSize(tEncodeSTqCheckInfo, &info, len, code);
if (code != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto END;
}
buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
MND_TMQ_NULL_CHECK(buf);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
SEncoder encoder;
SEncoder encoder = {0};
tEncoderInit(&encoder, abuf, len);
code = tEncodeSTqCheckInfo(&encoder, &info);
if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tEncoderClear(&encoder);
goto END;
}
tEncoderClear(&encoder);
@ -398,10 +406,7 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj
action.pCont = buf;
action.contLen = sizeof(SMsgHead) + len;
action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO;
code = mndTransAppendRedoAction(pTrans, &action);
if (code != 0) {
goto END;
}
MND_TMQ_RETURN_CHECK(mndTransAppendRedoAction(pTrans, &action));
sdbRelease(pSdb, pVgroup);
buf = NULL;
}
@ -417,33 +422,22 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
const char *userName) {
mInfo("start to create topic:%s", pCreate->name);
STrans *pTrans = NULL;
int32_t code = -1;
int32_t code = 0;
SNode *pAst = NULL;
SQueryPlan *pPlan = NULL;
SMqTopicObj topicObj = {0};
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic");
if (pTrans == NULL) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
code = -1;
goto _OUT;
}
MND_TMQ_NULL_CHECK(pTrans);
mndTransSetDbName(pTrans, pDb->name, NULL);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto _OUT;
}
MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
mInfo("trans:%d to create topic:%s", pTrans->id, pCreate->name);
tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN);
code = mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj);
if (code != 0) {
goto _OUT;
}
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj));
topicObj.createTime = taosGetTimestampMs();
topicObj.updateTime = topicObj.createTime;
@ -458,56 +452,25 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
topicObj.ast = taosStrdup(pCreate->ast);
topicObj.astLen = strlen(pCreate->ast) + 1;
qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast);
code = nodesStringToNode(pCreate->ast, &pAst);
if (code != 0) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
goto _OUT;
}
MND_TMQ_RETURN_CHECK(nodesStringToNode(pCreate->ast, &pAst));
SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true};
code = qCreateQueryPlan(&cxt, &pPlan, NULL);
if (code != 0) {
mError("failed to create topic:%s since %s", pCreate->name, terrstr());
goto _OUT;
}
MND_TMQ_RETURN_CHECK(qCreateQueryPlan(&cxt, &pPlan, NULL));
topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t));
if (topicObj.ntbColIds == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _OUT;
}
extractTopicTbInfo(pAst, &topicObj);
MND_TMQ_NULL_CHECK(topicObj.ntbColIds);
MND_TMQ_RETURN_CHECK(extractTopicTbInfo(pAst, &topicObj));
if (topicObj.ntbUid == 0) {
taosArrayDestroy(topicObj.ntbColIds);
(void)taosArrayDestroy(topicObj.ntbColIds);
topicObj.ntbColIds = NULL;
}
code = qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema);
if (code != 0) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
goto _OUT;
}
code = nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL);
if (code != 0) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
goto _OUT;
}
MND_TMQ_RETURN_CHECK(qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema));
MND_TMQ_RETURN_CHECK(nodesNodeToString((SNode *)pPlan, false, &topicObj.physicalPlan, NULL));
} else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) {
SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName);
if (pStb == NULL) {
terrno = TSDB_CODE_MND_STB_NOT_EXIST;
code = terrno;
goto _OUT;
}
strcpy(topicObj.stbName, pCreate->subStbName);
MND_TMQ_NULL_CHECK(pStb);
tstrncpy(topicObj.stbName, pCreate->subStbName, TSDB_TABLE_FNAME_LEN);
topicObj.stbUid = pStb->uid;
mndReleaseStb(pMnode, pStb);
if(pCreate->ast != NULL){
@ -518,34 +481,25 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
}
SSdbRaw *pCommitRaw = mndTopicActionEncode(&topicObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
code = -1;
goto _OUT;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (topicObj.ntbUid != 0) {
code = sendCheckInfoToVnode(pTrans, pMnode, &topicObj);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if(code != 0) {
goto _OUT;
}
sdbFreeRaw(pCommitRaw);
goto END;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
code = -1;
goto _OUT;
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
if (topicObj.ntbUid != 0) {
MND_TMQ_RETURN_CHECK(sendCheckInfoToVnode(pTrans, pMnode, &topicObj));
}
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
code = TSDB_CODE_ACTION_IN_PROGRESS;
_OUT:
END:
taosMemoryFreeClear(topicObj.physicalPlan);
taosMemoryFreeClear(topicObj.sql);
taosMemoryFreeClear(topicObj.ast);
taosArrayDestroy(topicObj.ntbColIds);
(void)taosArrayDestroy(topicObj.ntbColIds);
if (topicObj.schema.nCols) {
taosMemoryFreeClear(topicObj.schema.pSchema);
}
@ -557,75 +511,64 @@ _OUT:
static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
int32_t code = TDB_CODE_SUCCESS;
SMqTopicObj *pTopic = NULL;
SDbObj *pDb = NULL;
SCMCreateTopicReq createTopicReq = {0};
if (tDeserializeSCMCreateTopicReq(pReq->pCont, pReq->contLen, &createTopicReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
code = TSDB_CODE_INVALID_MSG;
goto END;
}
mInfo("topic:%s start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
if (mndCheckCreateTopicReq(&createTopicReq) != 0) {
mError("topic:%s failed to create since %s", createTopicReq.name, terrstr());
goto _OVER;
}
MND_TMQ_RETURN_CHECK(mndCheckCreateTopicReq(&createTopicReq));
pTopic = mndAcquireTopic(pMnode, createTopicReq.name);
if (pTopic != NULL) {
code = mndAcquireTopic(pMnode, createTopicReq.name, &pTopic);
if (code == TDB_CODE_SUCCESS) {
if (createTopicReq.igExists) {
mInfo("topic:%s already exist, ignore exist is set", createTopicReq.name);
code = 0;
goto _OVER;
goto END;
} else {
terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
goto _OVER;
code = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
goto END;
}
} else if (terrno != TSDB_CODE_MND_TOPIC_NOT_EXIST) {
goto _OVER;
} else if (code != TSDB_CODE_MND_TOPIC_NOT_EXIST) {
goto END;
}
pDb = mndAcquireDb(pMnode, createTopicReq.subDbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
goto _OVER;
}
MND_TMQ_NULL_CHECK(pDb);
if (pDb->cfg.walRetentionPeriod == 0) {
terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO;
code = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO;
mError("db:%s, not allowed to create topic when WAL_RETENTION_PERIOD is zero", pDb->name);
goto _OVER;
goto END;
}
if (sdbGetSize(pMnode->pSdb, SDB_TOPIC) >= tmqMaxTopicNum){
terrno = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE;
code = TSDB_CODE_TMQ_TOPIC_OUT_OF_RANGE;
mError("topic num out of range");
goto _OVER;
}
if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) {
goto _OVER;
goto END;
}
MND_TMQ_RETURN_CHECK(grantCheck(TSDB_GRANT_SUBSCRIPTION));
code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user);
if (code == 0) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
}
{
SName dbname = {0};
tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB);
(void)tNameFromString(&dbname, createTopicReq.subDbName, T_NAME_ACCT | T_NAME_DB); // ignore error
SName topicName = {0};
tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
//reuse this function for topic
(void)tNameFromString(&topicName, createTopicReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error
auditRecord(pReq, pMnode->clusterId, "createTopic", dbname.dbname, topicName.dbname,
createTopicReq.sql, strlen(createTopicReq.sql));
}
_OVER:
END:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("failed to create topic:%s since %s", createTopicReq.name, terrstr());
}
@ -638,22 +581,20 @@ _OVER:
}
static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) {
int32_t code = -1;
if (mndUserRemoveTopic(pMnode, pTrans, pTopic->name) != 0) {
goto _OVER;
int32_t code = 0;
SSdbRaw *pCommitRaw = NULL;
MND_TMQ_RETURN_CHECK(mndUserRemoveTopic(pMnode, pTrans, pTopic->name));
pCommitRaw = mndTopicActionEncode(pTopic);
MND_TMQ_NULL_CHECK(pCommitRaw);
code = mndTransAppendCommitlog(pTrans, pCommitRaw);
if(code != 0) {
sdbFreeRaw(pCommitRaw);
goto END;
}
MND_TMQ_RETURN_CHECK(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans));
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
goto _OVER;
}
code = 0;
_OVER:
END:
return code;
}
@ -661,7 +602,7 @@ static bool checkTopic(SArray *topics, char *topicName){
int32_t sz = taosArrayGetSize(topics);
for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(topics, i);
if (strcmp(name, topicName) == 0) {
if (name && strcmp(name, topicName) == 0) {
return true;
}
}
@ -682,29 +623,26 @@ static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topi
bool found = checkTopic(pConsumer->assignedTopics, topicName);
if (found){
if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) {
code = mndSetConsumerDropLogs(pTrans, pConsumer);
if (code != 0) {
goto end;
}
MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumer));
sdbRelease(pSdb, pConsumer);
continue;
}
mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s",
topicName, pConsumer->consumerId, pConsumer->cgroup);
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
goto end;
goto END;
}
if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) {
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)",
topicName, pConsumer->consumerId, pConsumer->cgroup);
goto end;
goto END;
}
sdbRelease(pSdb, pConsumer);
}
end:
END:
sdbRelease(pSdb, pConsumer);
sdbCancelFetch(pSdb, pIter);
return code;
@ -726,13 +664,10 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO
}
buf = taosMemoryCalloc(1, sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN);
if (buf == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
MND_TMQ_NULL_CHECK(buf);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN);
(void)memcpy(abuf, pTopic->name, TSDB_TOPIC_FNAME_LEN);
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
@ -742,12 +677,12 @@ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicO
code = mndTransAppendRedoAction(pTrans, &action);
if (code != 0) {
taosMemoryFree(buf);
goto end;
goto END;
}
sdbRelease(pSdb, pVgroup);
}
end:
END:
sdbRelease(pSdb, pVgroup);
sdbCancelFetch(pSdb, pIter);
return code;
@ -761,70 +696,40 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
STrans *pTrans = NULL;
if (tDeserializeSMDropTopicReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
return TSDB_CODE_INVALID_MSG;
}
pTopic = mndAcquireTopic(pMnode, dropReq.name);
if (pTopic == NULL) {
code = mndAcquireTopic(pMnode, dropReq.name, &pTopic);
if (code != 0) {
if (dropReq.igNotExists) {
mInfo("topic:%s, not exist, ignore not exist is set", dropReq.name);
tFreeSMDropTopicReq(&dropReq);
return 0;
} else {
terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
mError("topic:%s, failed to drop since %s", dropReq.name, terrstr());
tFreeSMDropTopicReq(&dropReq);
return -1;
return code;
}
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic");
if (pTrans == NULL) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
code = -1;
goto end;
}
MND_TMQ_NULL_CHECK(pTrans);
mndTransSetDbName(pTrans, pTopic->db, NULL);
code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) {
goto end;
}
MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans));
mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name);
code = mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic);
if (code != 0) {
goto end;
}
code = mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db);
if (code != 0) {
goto end;
}
code = mndDropConsumerByTopic(pMnode, pTrans, dropReq.name);
if (code != 0) {
goto end;
}
code = mndDropSubByTopic(pMnode, pTrans, dropReq.name);
if (code < 0) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
goto end;
}
MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic));
MND_TMQ_RETURN_CHECK(mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db));
MND_TMQ_RETURN_CHECK(mndDropConsumerByTopic(pMnode, pTrans, dropReq.name));
MND_TMQ_RETURN_CHECK(mndDropSubByTopic(pMnode, pTrans, dropReq.name));
if (pTopic->ntbUid != 0) {
code = mndDropCheckInfoByTopic(pMnode, pTrans, pTopic);
if (code != 0) {
goto end;
}
MND_TMQ_RETURN_CHECK(mndDropCheckInfoByTopic(pMnode, pTrans, pTopic));
}
code = mndDropTopic(pMnode, pTrans, pReq, pTopic);
end:
END:
mndReleaseTopic(pMnode, pTopic);
mndTransDrop(pTrans);
if (code != 0) {
@ -834,8 +739,7 @@ end:
}
SName name = {0};
tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
(void)tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // ignore error
auditRecord(pReq, pMnode->clusterId, "dropTopic", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen);
tFreeSMDropTopicReq(&dropReq);
@ -849,8 +753,7 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
return TSDB_CODE_MND_DB_NOT_SELECTED;
}
int32_t numOfTopics = 0;
@ -876,17 +779,23 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) {
static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
char* string = NULL;
int32_t code = 0;
cJSON* columns = cJSON_CreateArray();
if (columns == NULL) {
return;
}
MND_TMQ_NULL_CHECK(columns);
for (int i = 0; i < nCols; i++) {
cJSON* column = cJSON_CreateObject();
MND_TMQ_NULL_CHECK(column);
SSchema* s = schema + i;
cJSON* cname = cJSON_CreateString(s->name);
cJSON_AddItemToObject(column, "name", cname);
MND_TMQ_NULL_CHECK(cname);
if (!cJSON_AddItemToObject(column, "name", cname)) {
return;
}
cJSON* ctype = cJSON_CreateString(tDataTypes[s->type].name);
cJSON_AddItemToObject(column, "type", ctype);
MND_TMQ_NULL_CHECK(ctype);
if (!cJSON_AddItemToObject(column, "type", ctype)) {
return;
}
int32_t length = 0;
if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY ||
s->type == TSDB_DATA_TYPE_GEOMETRY) {
@ -897,8 +806,13 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
length = s->bytes;
}
cJSON* cbytes = cJSON_CreateNumber(length);
cJSON_AddItemToObject(column, "length", cbytes);
cJSON_AddItemToArray(columns, column);
MND_TMQ_NULL_CHECK(cbytes);
if (!cJSON_AddItemToObject(column, "length", cbytes)){
return;
}
if (!cJSON_AddItemToArray(columns, column)){
return;
}
}
string = cJSON_PrintUnformatted(columns);
cJSON_Delete(columns);
@ -910,6 +824,9 @@ static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){
mError("mndRetrieveTopic build schema error json:%p, json len:%zu", string, len);
}
taosMemoryFree(string);
END:
return;
}
static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
@ -917,13 +834,16 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SMqTopicObj *pTopic = NULL;
int32_t code = 0;
char *sql = NULL;
char *schemaJson = NULL;
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
SName n;
SColumnInfoData *pColInfo= NULL;
SName n = {0};
int32_t cols = 0;
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0};
@ -931,28 +851,34 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
STR_TO_VARSTR(topicName, pName);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false));
char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&n, varDataVal(dbName));
MND_TMQ_RETURN_CHECK(tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB));
MND_TMQ_RETURN_CHECK(tNameGetDbName(&n, varDataVal(dbName)));
varDataSetLen(dbName, strlen(varDataVal(dbName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)dbName, false));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)&pTopic->createTime, false));
char *sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE);
sql = taosMemoryMalloc(strlen(pTopic->sql) + VARSTR_HEADER_SIZE);
MND_TMQ_NULL_CHECK(sql);
STR_TO_VARSTR(sql, pTopic->sql);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)sql, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)sql, false));
taosMemoryFree(sql);
taosMemoryFreeClear(sql);
char *schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE);
schemaJson = taosMemoryMalloc(TSDB_SHOW_SCHEMA_JSON_LEN + VARSTR_HEADER_SIZE);
MND_TMQ_NULL_CHECK(schemaJson);
if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){
schemaToJson(pTopic->schema.pSchema, pTopic->schema.nCols, schemaJson);
}else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){
@ -969,8 +895,9 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false);
taosMemoryFree(schemaJson);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)schemaJson, false));
taosMemoryFreeClear(schemaJson);
char mete[4 + VARSTR_HEADER_SIZE] = {0};
if(pTopic->withMeta){
@ -980,7 +907,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)mete, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)mete, false));
char type[8 + VARSTR_HEADER_SIZE] = {0};
if(pTopic->subType == TOPIC_SUB_TYPE__COLUMN){
@ -992,7 +920,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
}
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)type, false);
MND_TMQ_NULL_CHECK(pColInfo);
MND_TMQ_RETURN_CHECK(colDataSetVal(pColInfo, numOfRows, (const char *)type, false));
numOfRows++;
sdbRelease(pSdb, pTopic);
@ -1000,24 +929,11 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
pShow->numOfRows += numOfRows;
return numOfRows;
}
int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) {
SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
END:
taosMemoryFreeClear(schemaJson);
taosMemoryFreeClear(sql);
return code;
}
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) {
@ -1047,33 +963,3 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) {
return false;
}
#if 0
int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
SMqTopicObj *pTopic = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
if (pIter == NULL) break;
if (pTopic->dbUid != pDb->uid) {
sdbRelease(pSdb, pTopic);
continue;
}
if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) < 0) {
sdbRelease(pSdb, pTopic);
sdbCancelFetch(pSdb, pIter);
code = -1;
break;
}
sdbRelease(pSdb, pTopic);
}
return code;
}
#endif

18
source/dnode/mnode/impl/src/mndUser.c
View File

@ -1357,7 +1357,7 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) {
}
int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
memcpy(pNew, pUser, sizeof(SUserObj));
(void)memcpy(pNew, pUser, sizeof(SUserObj));
pNew->authVersion++;
pNew->updateTime = taosGetTimestampMs();
@ -1377,7 +1377,7 @@ int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
taosRUnLockLatch(&pUser->lock);
if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
return 0;
}
@ -1983,10 +1983,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
int32_t len = strlen(pAlterReq->objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname);
if (pTopic == NULL) {
SMqTopicObj *pTopic = NULL;
int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic);
if (code != 0) {
mndReleaseTopic(pMnode, pTopic);
return -1;
return code;
}
taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
mndReleaseTopic(pMnode, pTopic);
@ -1994,10 +1995,11 @@ static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode
if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
int32_t len = strlen(pAlterReq->objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname);
if (pTopic == NULL) {
SMqTopicObj *pTopic = NULL;
int32_t code = mndAcquireTopic(pMnode, pAlterReq->objname, &pTopic);
if (code != 0) {
mndReleaseTopic(pMnode, pTopic);
return -1;
return code;
}
taosHashRemove(pNewUser->topics, pAlterReq->objname, len);
mndReleaseTopic(pMnode, pTopic);

9
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -2725,15 +2725,6 @@ int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgro
SDbObj dbObj = {0};
SArray *pArray = mndBuildDnodesArray(pMnode, 0);
// int32_t numOfTopics = 0;
// if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) {
// goto _OVER;
// }
// if (numOfTopics > 0) {
// terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED;
// goto _OVER;
// }
int32_t numOfStreams = 0;
if (mndGetNumOfStreams(pMnode, pDb->name, &numOfStreams) != 0) {
goto _OVER;

25
source/dnode/qnode/src/qnode.c
View File

@ -14,26 +14,28 @@
*/
#include "tqueue.h"
#include "executor.h"
#include "qndInt.h"
#include "query.h"
#include "qworker.h"
SQnode *qndOpen(const SQnodeOpt *pOption) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
if (NULL == pQnode) {
int32_t qndOpen(const SQnodeOpt *pOption, SQnode **pQnode) {
*pQnode = taosMemoryCalloc(1, sizeof(SQnode));
if (NULL == *pQnode) {
qError("calloc SQnode failed");
return NULL;
return terrno;
}
pQnode->qndId = QNODE_HANDLE;
(*pQnode)->qndId = QNODE_HANDLE;
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, (void **)&pQnode->pQuery, &pOption->msgCb)) {
int32_t code = qWorkerInit(NODE_TYPE_QNODE, (*pQnode)->qndId, (void **)&(*pQnode)->pQuery, &pOption->msgCb);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFreeClear(pQnode);
return NULL;
return code;
}
pQnode->msgCb = pOption->msgCb;
return pQnode;
(*pQnode)->msgCb = pOption->msgCb;
return TSDB_CODE_SUCCESS;
}
void qndClose(SQnode *pQnode) {
@ -112,6 +114,9 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, SQueueInfo* pInfo, SRpcMsg *pMsg) {
terrno = TSDB_CODE_APP_ERROR;
}
if (code == 0) return TSDB_CODE_ACTION_IN_PROGRESS;
if (code == 0) {
return TSDB_CODE_ACTION_IN_PROGRESS;
} else {
return code;
}
}

1
source/dnode/snode/src/snode.c
View File

@ -64,7 +64,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
int32_t code = 0;
SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode));
if (pSnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}

1
source/dnode/vnode/CMakeLists.txt
View File

@ -22,7 +22,6 @@ set(
# meta
"src/meta/metaOpen.c"
"src/meta/metaIdx.c"
"src/meta/metaTable.c"
"src/meta/metaSma.c"
"src/meta/metaQuery.c"

2
source/dnode/vnode/inc/vnode.h
View File

@ -185,7 +185,7 @@ int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList,
SArray *pFuncTypeList, SColumnInfo *pkCol, int32_t numOfPks);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);
void tsdbCacherowsReaderClose(void *pReader);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);

4
source/dnode/vnode/src/inc/tq.h
View File

@ -155,8 +155,8 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_
SSubmitTbData* pTableData, int64_t earlyTs, const char* id);
int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule);
int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq);
#define TQ_ERR_GO_TO_END(c) \
do { \

4
source/dnode/vnode/src/inc/tsdb.h
View File

@ -226,7 +226,7 @@ void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32
int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *),
int32_t (*tItemCmprFn)(const void *, const void *), void *pItem);
int32_t tPutMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData);
int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize);
int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItemFn)(uint8_t *, void *),
SArray **ppArray);
// other
@ -244,7 +244,6 @@ void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive);
STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode);
int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive);
SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable);
// STbDataIter
int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter **ppIter);
void *tsdbTbDataIterDestroy(STbDataIter *pIter);
@ -706,7 +705,6 @@ typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList;
STsdbFSetPartList *tsdbFSetPartListCreate();
void tsdbFSetPartListDestroy(STsdbFSetPartList **ppList);
int32_t tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
int32_t tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges);

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -318,7 +318,7 @@ void* tdUidStoreFree(STbUidStore* pStore);
// SMetaSnapReader ========================================
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
void metaSnapReaderClose(SMetaSnapReader** ppReader);
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
// SMetaSnapWriter ========================================
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);

133
source/dnode/vnode/src/meta/metaCache.c
View File

@ -118,90 +118,80 @@ static void freeCacheEntryFp(void* param) {
int32_t metaCacheOpen(SMeta* pMeta) {
int32_t code = 0;
SMetaCache* pCache = NULL;
int32_t lino;
pCache = (SMetaCache*)taosMemoryMalloc(sizeof(SMetaCache));
if (pCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
pMeta->pCache = (SMetaCache*)taosMemoryCalloc(1, sizeof(SMetaCache));
if (pMeta->pCache == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// open entry cache
pCache->sEntryCache.nEntry = 0;
pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET;
pCache->sEntryCache.aBucket =
(SMetaCacheEntry**)taosMemoryCalloc(pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*));
if (pCache->sEntryCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
pMeta->pCache->sEntryCache.nEntry = 0;
pMeta->pCache->sEntryCache.nBucket = META_CACHE_BASE_BUCKET;
pMeta->pCache->sEntryCache.aBucket =
(SMetaCacheEntry**)taosMemoryCalloc(pMeta->pCache->sEntryCache.nBucket, sizeof(SMetaCacheEntry*));
if (pMeta->pCache->sEntryCache.aBucket == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
// open stats cache
pCache->sStbStatsCache.nEntry = 0;
pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET;
pCache->sStbStatsCache.aBucket =
(SMetaStbStatsEntry**)taosMemoryCalloc(pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*));
if (pCache->sStbStatsCache.aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->sStbStatsCache.nEntry = 0;
pMeta->pCache->sStbStatsCache.nBucket = META_CACHE_STATS_BUCKET;
pMeta->pCache->sStbStatsCache.aBucket =
(SMetaStbStatsEntry**)taosMemoryCalloc(pMeta->pCache->sStbStatsCache.nBucket, sizeof(SMetaStbStatsEntry*));
if (pMeta->pCache->sStbStatsCache.aBucket == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->sTagFilterResCache.pUidResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pMeta->pCache->sTagFilterResCache.pUidResCache == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pCache->sTagFilterResCache.accTimes = 0;
pCache->sTagFilterResCache.pTableEntry =
pMeta->pCache->sTagFilterResCache.accTimes = 0;
pMeta->pCache->sTagFilterResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->sTagFilterResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
if (pMeta->pCache->sTagFilterResCache.pTableEntry == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
taosHashSetFreeFp(pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->sTagFilterResCache.lock, NULL);
taosHashSetFreeFp(pMeta->pCache->sTagFilterResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pMeta->pCache->sTagFilterResCache.lock, NULL);
pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pCache->STbGroupResCache.pResCache == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->STbGroupResCache.pResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5);
if (pMeta->pCache->STbGroupResCache.pResCache == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pCache->STbGroupResCache.accTimes = 0;
pCache->STbGroupResCache.pTableEntry =
pMeta->pCache->STbGroupResCache.accTimes = 0;
pMeta->pCache->STbGroupResCache.pTableEntry =
taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbGroupResCache.pTableEntry == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
if (pMeta->pCache->STbGroupResCache.pTableEntry == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL);
taosHashSetFreeFp(pMeta->pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pMeta->pCache->STbGroupResCache.lock, NULL);
pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStb == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
pMeta->pCache->STbFilterCache.pStb =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pMeta->pCache->STbFilterCache.pStb == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pCache->STbFilterCache.pStbName =
pMeta->pCache->STbFilterCache.pStbName =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStbName == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
if (pMeta->pCache->STbFilterCache.pStbName == NULL) {
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
pMeta->pCache = pCache;
return code;
_err2:
entryCacheClose(pMeta);
_err:
taosMemoryFree(pCache);
metaError("vgId:%d, meta open cache failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
metaCacheClose(pMeta);
} else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
}
return code;
}
@ -289,8 +279,7 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) {
}
} else { // insert
if (pCache->sEntryCache.nEntry >= pCache->sEntryCache.nBucket) {
code = metaRehashCache(pCache, 1);
if (code) goto _exit;
TAOS_UNUSED(metaRehashCache(pCache, 1));
iBucket = TABS(pInfo->uid) % pCache->sEntryCache.nBucket;
}
@ -328,8 +317,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sEntryCache.nEntry--;
if (pCache->sEntryCache.nEntry < pCache->sEntryCache.nBucket / 4 &&
pCache->sEntryCache.nBucket > META_CACHE_BASE_BUCKET) {
code = metaRehashCache(pCache, 0);
if (code) goto _exit;
TAOS_UNUSED(metaRehashCache(pCache, 0));
}
} else {
code = TSDB_CODE_NOT_FOUND;
@ -351,7 +339,9 @@ int32_t metaCacheGet(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo) {
}
if (pEntry) {
if (pInfo) {
*pInfo = pEntry->info;
}
} else {
code = TSDB_CODE_NOT_FOUND;
}
@ -415,9 +405,7 @@ int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) {
(*ppEntry)->info.ctbNum = pInfo->ctbNum;
} else { // insert
if (pCache->sStbStatsCache.nEntry >= pCache->sStbStatsCache.nBucket) {
code = metaRehashStatsCache(pCache, 1);
if (code) goto _exit;
TAOS_UNUSED(metaRehashStatsCache(pCache, 1));
iBucket = TABS(pInfo->uid) % pCache->sStbStatsCache.nBucket;
}
@ -454,8 +442,7 @@ int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid) {
pCache->sStbStatsCache.nEntry--;
if (pCache->sStbStatsCache.nEntry < pCache->sStbStatsCache.nBucket / 4 &&
pCache->sStbStatsCache.nBucket > META_CACHE_STATS_BUCKET) {
code = metaRehashStatsCache(pCache, 0);
if (code) goto _exit;
TAOS_UNUSED(metaRehashStatsCache(pCache, 0));
}
} else {
code = TSDB_CODE_NOT_FOUND;
@ -477,7 +464,9 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) {
}
if (pEntry) {
if (pInfo) {
*pInfo = pEntry->info;
}
} else {
code = TSDB_CODE_NOT_FOUND;
}
@ -502,7 +491,9 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv
// check whether it is existed in LRU cache, and remove it from linked list if not.
LRUHandle* pRes = taosLRUCacheLookup(pCache, buf, len);
if (pRes == NULL) { // remove the item in the linked list
taosArrayPush(pInvalidRes, &pNode);
if (taosArrayPush(pInvalidRes, &pNode) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
taosLRUCacheRelease(pCache, pRes, false);
}
@ -626,7 +617,7 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL
p->hitTimes = 0;
tdListInit(&p->list, keyLen);
taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES);
TAOS_CHECK_RETURN(taosHashPut(pTableEntry, &suid, sizeof(uint64_t), &p, POINTER_BYTES));
tdListAppend(&p->list, pKey);
return 0;
}
@ -956,9 +947,7 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) {
}
if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) {
return terrno ? terrno : -1;
}
TAOS_CHECK_RETURN(metaPutTbToFilterCache(pMeta, pTbArr[i], 1));
}
}
#else

42
source/dnode/vnode/src/meta/metaCommit.c
View File

@ -21,7 +21,10 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) {
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
// begin a meta txn
int metaBegin(SMeta *pMeta, int8_t heap) {
int32_t metaBegin(SMeta *pMeta, int8_t heap) {
int32_t code = 0;
int32_t lino;
void *(*xMalloc)(void *, size_t) = NULL;
void (*xFree)(void *, void *) = NULL;
void *xArg = NULL;
@ -36,12 +39,19 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
xArg = pMeta->pVnode->inUse;
}
if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
return -1;
code = tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
TSDB_CHECK_CODE(code, lino, _exit);
code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
}
tdbCommit(pMeta->pEnv, pMeta->txn);
return 0;
}
@ -49,20 +59,36 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; }
int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); }
int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); }
int metaPrepareAsyncCommit(SMeta *pMeta) {
// return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
int code = 0;
int32_t lino;
metaWLock(pMeta);
code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
TAOS_UNUSED(ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn));
metaULock(pMeta);
code = tdbCommit(pMeta->pEnv, pMeta->txn);
TSDB_CHECK_CODE(code, lino, _exit);
pMeta->changed = false;
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, __LINE__,
tstrerror(terrno));
} else {
metaDebug("vgId:%d %s success", TD_VID(pMeta->pVnode), __func__);
}
return code;
}
// abort the meta txn
int metaAbort(SMeta *pMeta) {
if (!pMeta->txn) return 0;
if (!pMeta->txn) {
return 0;
}
int code = tdbAbort(pMeta->pEnv, pMeta->txn);
if (code) {
metaError("vgId:%d, failed to abort meta since %s", TD_VID(pMeta->pVnode), tstrerror(terrno));

143
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -17,159 +17,166 @@
int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) {
const SColCmprWrapper *pw = &pME->colCmpr;
if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1;
if (tEncodeI32v(pCoder, pw->version) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->nCols));
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pw->version));
uDebug("encode cols:%d", pw->nCols);
for (int32_t i = 0; i < pw->nCols; i++) {
SColCmpr *p = &pw->pColCmpr[i];
if (tEncodeI16v(pCoder, p->id) < 0) return -1;
if (tEncodeU32(pCoder, p->alg) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI16v(pCoder, p->id));
TAOS_CHECK_RETURN(tEncodeU32(pCoder, p->alg));
}
return 0;
}
int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) {
SColCmprWrapper *pWrapper = &pME->colCmpr;
if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->nCols));
if (pWrapper->nCols == 0) {
return 0;
}
if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32v(pDecoder, &pWrapper->version));
uDebug("dencode cols:%d", pWrapper->nCols);
pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr));
if (pWrapper->pColCmpr == NULL) return -1;
if (pWrapper->pColCmpr == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < pWrapper->nCols; i++) {
SColCmpr *p = &pWrapper->pColCmpr[i];
if (tDecodeI16v(pDecoder, &p->id) < 0) return -1;
if (tDecodeU32(pDecoder, &p->alg) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI16v(pDecoder, &p->id));
TAOS_CHECK_RETURN(tDecodeU32(pDecoder, &p->alg));
}
return 0;
}
static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
static FORCE_INLINE int32_t metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
SSchemaWrapper *pSchema) {
pCmpr->nCols = pSchema->nCols;
pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr));
if ((pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr *pColCmpr = &pCmpr->pColCmpr[i];
SSchema *pColSchema = &pSchema->pSchema[i];
pColCmpr->id = pColSchema->colId;
pColCmpr->alg = createDefaultColCmprByType(pColSchema->type);
}
return 0;
}
int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1;
TAOS_CHECK_RETURN(tStartEncode(pCoder));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->version));
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->type));
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->uid));
if (tEncodeI64(pCoder, pME->version) < 0) return -1;
if (tEncodeI8(pCoder, pME->type) < 0) return -1;
if (tEncodeI64(pCoder, pME->uid) < 0) return -1;
if (pME->name == NULL || tEncodeCStr(pCoder, pME->name) < 0) return -1;
if (pME->name == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->name));
if (pME->type == TSDB_SUPER_TABLE) {
if (tEncodeI8(pCoder, pME->flags) < 0) return -1; // TODO: need refactor?
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI8(pCoder, pME->flags));
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow));
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) {
if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
}
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.btime));
TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ctbEntry.ttlDays));
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ctbEntry.comment));
}
if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ctbEntry.suid));
TAOS_CHECK_RETURN(tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1;
if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI64(pCoder, pME->ntbEntry.btime));
TAOS_CHECK_RETURN(tEncodeI32(pCoder, pME->ntbEntry.ttlDays));
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) {
if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeCStr(pCoder, pME->ntbEntry.comment));
}
if (tEncodeI32v(pCoder, pME->ntbEntry.ncid) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeI32v(pCoder, pME->ntbEntry.ncid));
TAOS_CHECK_RETURN(tEncodeSSchemaWrapper(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) {
if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1;
TAOS_CHECK_RETURN(tEncodeTSma(pCoder, pME->smaEntry.tsma));
} else {
metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1;
TAOS_CHECK_RETURN(meteEncodeColCmprEntry(pCoder, pME));
tEndEncode(pCoder);
return 0;
}
int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pME->version) < 0) return -1;
if (tDecodeI8(pCoder, &pME->type) < 0) return -1;
if (tDecodeI64(pCoder, &pME->uid) < 0) return -1;
if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
TAOS_CHECK_RETURN(tStartDecode(pCoder));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->version));
TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->type));
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->uid));
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->name));
if (pME->type == TSDB_SUPER_TABLE) {
if (tDecodeI8(pCoder, &pME->flags) < 0) return -1; // TODO: need refactor?
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI8(pCoder, &pME->flags));
TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow));
TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag));
if (TABLE_IS_ROLLUP(pME->flags)) {
if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam));
}
} else if (pME->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.btime));
TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ctbEntry.ttlDays));
TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ctbEntry.commentLen));
if (pME->ctbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ctbEntry.comment));
}
if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO)
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ctbEntry.suid));
TAOS_CHECK_RETURN(tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags));
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1;
if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI64(pCoder, &pME->ntbEntry.btime));
TAOS_CHECK_RETURN(tDecodeI32(pCoder, &pME->ntbEntry.ttlDays));
TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.commentLen));
if (pME->ntbEntry.commentLen > 0) {
if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeCStr(pCoder, &pME->ntbEntry.comment));
}
if (tDecodeI32v(pCoder, &pME->ntbEntry.ncid) < 0) return -1;
if (tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeI32v(pCoder, &pME->ntbEntry.ncid));
TAOS_CHECK_RETURN(tDecodeSSchemaWrapperEx(pCoder, &pME->ntbEntry.schemaRow));
} else if (pME->type == TSDB_TSMA_TABLE) {
pME->smaEntry.tsma = tDecoderMalloc(pCoder, sizeof(STSma));
if (!pME->smaEntry.tsma) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
return TSDB_CODE_OUT_OF_MEMORY;
}
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1;
TAOS_CHECK_RETURN(tDecodeTSma(pCoder, pME->smaEntry.tsma, true));
} else {
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (pME->type == TSDB_SUPER_TABLE) {
if (TABLE_IS_COL_COMPRESSED(pME->flags)) {
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
}
} else {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow));
TABLE_SET_COL_COMPRESSED(pME->flags);
}
} else if (pME->type == TSDB_NORMAL_TABLE) {
if (!tDecodeIsEnd(pCoder)) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name);
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
TAOS_CHECK_RETURN(meteDecodeColCmprEntry(pCoder, pME));
if (pME->colCmpr.nCols == 0) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
}
} else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow);
TAOS_CHECK_RETURN(metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow));
}
TABLE_SET_COL_COMPRESSED(pME->flags);
}

118
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -1,118 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef USE_INVERTED_INDEX
#include "index.h"
#endif
#include "meta.h"
struct SMetaIdx {
#ifdef USE_INVERTED_INDEX
SIndex *pIdx;
#endif
/* data */
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
};
int metaOpenIdx(SMeta *pMeta) {
#if 0
char idxDir[128]; // TODO
char * err = NULL;
rocksdb_options_t *options = rocksdb_options_create();
// TODO
sprintf(idxDir, "%s/index", pMeta->path);
if (pMeta->pCache) {
rocksdb_options_set_row_cache(options, pMeta->pCache);
}
rocksdb_options_set_create_if_missing(options, 1);
pMeta->pIdx = rocksdb_open(options, idxDir, &err);
if (pMeta->pIdx == NULL) {
// TODO: handle error
rocksdb_options_destroy(options);
return -1;
}
rocksdb_options_destroy(options);
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexOpen(&opts, pMeta->path, &pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
#endif
return 0;
}
#ifdef BUILD_NO_CALL
void metaCloseIdx(SMeta *pMeta) { /* TODO */
#if 0
if (pMeta->pIdx) {
rocksdb_close(pMeta->pIdx);
pMeta->pIdx = NULL;
}
#endif
#ifdef USE_INVERTED_INDEX
// SIndexOpts opts;
// if (indexClose(pMeta->pIdx->pIdx) != 0) {
// return -1;
//}
// return 0;
#endif
}
int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbCfg) {
#ifdef USE_INVERTED_INDEX
// if (pTbCfgs->type == META_CHILD_TABLE) {
// char buf[8] = {0};
// int16_t colId = (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId;
// sprintf(buf, "%d", colId); // colname
// char *pTagVal = (char *)tdGetKVRowValOfCol(pTbCfg->ctbCfg.pTag, (kvRowColIdx(pTbCfg->ctbCfg.pTag))[0].colId);
// tb_uid_t suid = pTbCfg->ctbCfg.suid; // super id
// tb_uid_t tuid = 0; // child table uid
// SIndexMultiTerm *terms = indexMultiTermCreate();
// SIndexTerm *term =
// indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_BINARY, buf, strlen(buf), pTagVal, strlen(pTagVal), tuid);
// indexMultiTermAdd(terms, term);
// int ret = indexPut(pMeta->pIdx->pIdx, terms);
// indexMultiTermDestroy(terms);
// return ret;
//} else {
// return DB_DONOTINDEX;
//}
#endif
// TODO
return 0;
}
int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
#ifdef USE_INVERTED_INDEX
#endif
// TODO
return 0;
}
#endif

209
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -39,13 +39,13 @@ static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&p
static void metaCleanup(SMeta **ppMeta);
int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
int32_t metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
SMeta *pMeta = NULL;
int ret;
int offset;
int32_t code = 0;
int32_t lino;
int32_t offset;
char path[TSDB_FILENAME_LEN] = {0};
*ppMeta = NULL;
char indexFullPath[128] = {0};
// create handle
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, path, TSDB_FILENAME_LEN);
@ -53,8 +53,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
snprintf(path + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VNODE_META_DIR);
if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + strlen(path) + 1)) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
metaInitLock(pMeta);
@ -69,163 +68,104 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
taosMkDir(pMeta->path);
// open env
ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
code = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv, rollback,
pVnode->config.tdbEncryptAlgorithm, pVnode->config.tdbEncryptKey);
if (ret < 0) {
metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
// open pTbDb
ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pSkmDb
ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pUidIdx
ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(SUidIdxVal), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pNameIdx
ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pCtbIdx
ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pSuidIdx
ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
char indexFullPath[128] = {0};
sprintf(indexFullPath, "%s/%s", pMeta->path, "invert");
taosMkDir(indexFullPath);
TAOS_UNUSED(taosMkDir(indexFullPath));
SIndexOpts opts = {.cacheSize = 8 * 1024 * 1024};
ret = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
if (ret < 0) {
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = indexOpen(&opts, indexFullPath, (SIndex **)&pMeta->pTagIvtIdx);
TSDB_CHECK_CODE(code, lino, _exit);
ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open pTtlMgr ("ttlv1.idx")
char logPrefix[128] = {0};
sprintf(logPrefix, "vgId:%d", TD_VID(pVnode));
ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold);
if (ret < 0) {
metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0, logPrefix, tsTtlFlushThreshold);
TSDB_CHECK_CODE(code, lino, _exit);
// open pSmaIdx
ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// idx table create time
ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// idx num of col, normal table only
ret = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta ncol index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("ncol.idx", sizeof(SNcolIdxKey), 0, ncolIdxCmpr, pMeta->pEnv, &pMeta->pNcolIdx, 0);
TSDB_CHECK_CODE(code, lino, _exit);
ret = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0);
if (ret < 0) {
metaError("vgId:%d, failed to open meta stream task index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = tdbTbOpen("stream.task.db", sizeof(int64_t), -1, taskIdxKeyCmpr, pMeta->pEnv, &pMeta->pStreamDb, 0);
TSDB_CHECK_CODE(code, lino, _exit);
// open index
if (metaOpenIdx(pMeta) < 0) {
metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
code = metaCacheOpen(pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t code = metaCacheOpen(pMeta);
code = metaInitTbFilterCache(pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
terrno = code;
metaError("vgId:%d, failed to open meta cache since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
if (metaInitTbFilterCache(pMeta) != 0) {
goto _err;
}
metaDebug("vgId:%d, meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;
return 0;
_err:
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaCleanup(&pMeta);
return -1;
*ppMeta = NULL;
} else {
metaDebug("vgId:%d %s success", TD_VID(pVnode), __func__);
*ppMeta = pMeta;
}
return code;
}
int metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
int code = TSDB_CODE_SUCCESS;
int32_t metaUpgrade(SVnode *pVnode, SMeta **ppMeta) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino;
SMeta *pMeta = *ppMeta;
if (ttlMgrNeedUpgrade(pMeta->pEnv)) {
code = metaBegin(pMeta, META_BEGIN_HEAP_OS);
if (code < 0) {
metaError("vgId:%d, failed to upgrade meta, meta begin failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
code = ttlMgrUpgrade(pMeta->pTtlMgr, pMeta);
if (code < 0) {
metaError("vgId:%d, failed to upgrade meta ttl since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
code = metaCommit(pMeta, pMeta->txn);
if (code < 0) {
metaError("vgId:%d, failed to upgrade meta ttl, meta commit failed since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
return TSDB_CODE_SUCCESS;
_err:
_exit:
if (code) {
metaError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
metaCleanup(ppMeta);
}
return code;
}
@ -235,33 +175,42 @@ int metaClose(SMeta **ppMeta) {
}
int metaAlterCache(SMeta *pMeta, int32_t nPage) {
int32_t code = 0;
metaWLock(pMeta);
if (tdbAlter(pMeta->pEnv, nPage) < 0) {
code = tdbAlter(pMeta->pEnv, nPage);
metaULock(pMeta);
return -1;
if (code) {
metaError("vgId:%d %s failed since %s", TD_VID(pMeta->pVnode), __func__, tstrerror(code));
}
metaULock(pMeta);
return 0;
return code;
}
int32_t metaRLock(SMeta *pMeta) {
metaTrace("meta rlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockRdlock(&pMeta->lock);
return ret;
int32_t code = taosThreadRwlockRdlock(&pMeta->lock);
if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
}
int32_t metaWLock(SMeta *pMeta) {
metaTrace("meta wlock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockWrlock(&pMeta->lock);
return ret;
int32_t code = taosThreadRwlockWrlock(&pMeta->lock);
if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
}
int32_t metaULock(SMeta *pMeta) {
metaTrace("meta ulock %p", &pMeta->lock);
int32_t ret = taosThreadRwlockUnlock(&pMeta->lock);
return ret;
int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
if (code) {
return TAOS_SYSTEM_ERROR(code);
}
return 0;
}
static void metaCleanup(SMeta **ppMeta) {

9
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1542,9 +1542,14 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t
// slow path: search TDB
int64_t ctbNum = 0;
int32_t colNum = 0;
vnodeGetCtbNum(pVnode, uid, &ctbNum);
vnodeGetStbColumnNum(pVnode, uid, &colNum);
code = vnodeGetCtbNum(pVnode, uid, &ctbNum);
if (TSDB_CODE_SUCCESS == code) {
code = vnodeGetStbColumnNum(pVnode, uid, &colNum);
}
metaULock(pVnodeObj->pMeta);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
if (numOfTables) *numOfTables = ctbNum;
if (numOfCols) *numOfCols = colNum;

86
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -25,14 +25,14 @@ struct SMetaSnapReader {
int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
int32_t code = 0;
int32_t lino;
int32_t c = 0;
SMetaSnapReader* pReader = NULL;
// alloc
pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pReader->pMeta = pMeta;
pReader->sver = sver;
@ -40,36 +40,29 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe
// impl
code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL);
if (code) {
taosMemoryFree(pReader);
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
taosMemoryFree(pReader);
goto _err;
}
metaInfo("vgId:%d, vnode snapshot meta reader opened", TD_VID(pMeta->pVnode));
*ppReader = pReader;
return code;
_err:
metaError("vgId:%d, vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
metaSnapReaderClose(&pReader);
*ppReader = NULL;
} else {
metaInfo("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppReader = pReader;
}
return code;
}
int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
int32_t code = 0;
void metaSnapReaderClose(SMetaSnapReader** ppReader) {
if (ppReader && *ppReader) {
tdbTbcClose((*ppReader)->pTbc);
taosMemoryFree(*ppReader);
*ppReader = NULL;
return code;
}
}
int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
@ -106,7 +99,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
goto _exit;
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
@ -122,10 +115,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
}
_exit:
return code;
_err:
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code));
if (code) {
metaError("vgId:%d, vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode),
tstrerror(code));
}
return code;
}
@ -138,26 +131,30 @@ struct SMetaSnapWriter {
int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
int32_t code = 0;
int32_t lino;
SMetaSnapWriter* pWriter;
// alloc
pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pWriter->pMeta = pMeta;
pWriter->sver = sver;
pWriter->ever = ever;
metaBegin(pMeta, META_BEGIN_HEAP_NIL);
code = metaBegin(pMeta, META_BEGIN_HEAP_NIL);
TSDB_CHECK_CODE(code, lino, _exit);
*ppWriter = pWriter;
return code;
_err:
metaError("vgId:%d, meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
taosMemoryFree(pWriter);
*ppWriter = NULL;
} else {
metaDebug("vgId:%d, %s success", TD_VID(pMeta->pVnode), __func__);
*ppWriter = pWriter;
}
return code;
}
@ -189,25 +186,24 @@ _err:
int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
int32_t line = 0;
int32_t lino = 0;
SMeta* pMeta = pWriter->pMeta;
SMetaEntry metaEntry = {0};
SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
code = metaDecodeEntry(pDecoder, &metaEntry);
VND_CHECK_CODE(code, line, _err);
TSDB_CHECK_CODE(code, lino, _exit);
code = metaHandleEntry(pMeta, &metaEntry);
VND_CHECK_CODE(code, line, _err);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
metaError("vgId:%d, %s failed at %s:%d since %s", TD_VID(pMeta->pVnode), __func__, __FILE__, lino, tstrerror(code));
}
tDecoderClear(pDecoder);
return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
return code;
}
typedef struct STableInfoForChildTable {
@ -478,9 +474,7 @@ void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid){
ctx->hasPrimaryKey = ret;
}
bool taosXGetTablePrimaryKey(SSnapContext* ctx){
return ctx->hasPrimaryKey;
}
bool taosXGetTablePrimaryKey(SSnapContext* ctx) { return ctx->hasPrimaryKey; }
int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid) {
int32_t ret = 0;

16
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -23,7 +23,9 @@ static int32_t rsmaRestore(SSma *pSma);
#define SMA_SET_KEEP_CFG(v, l) \
do { \
SRetention *r = &pCfg->retentions[l]; \
pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
int64_t keep = -1; \
convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep); \
pKeepCfg->keep2 = (int32_t)keep; \
pKeepCfg->keep0 = pKeepCfg->keep2; \
pKeepCfg->keep1 = pKeepCfg->keep2; \
pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days); \
@ -60,8 +62,12 @@ static int32_t rsmaRestore(SSma *pSma);
* @return int32_t
*/
static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) {
int32_t freqDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE);
int32_t keepDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE);
int32_t code = TSDB_CODE_SUCCESS;
int64_t freqDuration = -1;
int64_t keepDuration = -1;
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
code = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t days = duration; // min
if (days < freqDuration) {
@ -76,8 +82,8 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
goto _exit;
}
freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE);
keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE);
code = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration);
code = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration);
int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level
days *= (nFreqTimes > 1 ? nFreqTimes : 1);

7
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -335,8 +335,9 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
}
if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
int64_t msInterval =
convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND);
int64_t msInterval = -1;
TAOS_CHECK_RETURN(convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision,
TIME_UNIT_MILLISECOND, &msInterval));
pItem->maxDelay = (int32_t)msInterval;
} else {
pItem->maxDelay = (int32_t)param->maxdelay[idx];
@ -1441,7 +1442,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
} break;
case TASK_TRIGGER_STAT_INIT: {
smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid::%" PRIi64 " since stat is init",
smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is init",
SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
} break;
default: {

18
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -69,8 +69,9 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
}
STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg;
int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND);
if (sInterval <= 0) {
int64_t sInterval = -1;
code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND, &sInterval);
if (TSDB_CODE_SUCCESS != code || 0 == sInterval) {
*days = pTsdbCfg->days;
goto _exit;
}
@ -78,7 +79,11 @@ static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t c
if (records >= SMA_STORAGE_SPLIT_FACTOR) {
*days = pTsdbCfg->days;
} else {
int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE);
int64_t mInterval = -1;
code = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE, &mInterval);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2;
if (daysPerFile > SMA_STORAGE_MINUTES_MAX) {
@ -196,7 +201,12 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
code = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true, &tbData.pCreateTbReq);
if (code) {
smaError("failed to build create-table req, code:%d", code);
continue;
}
{
uint64_t groupId = pDataBlock->info.id.groupId;

20
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -194,12 +194,13 @@ END:
static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) {
int32_t code = TDB_CODE_SUCCESS;
SVnode* pVnode = pTq->pVnode;
int32_t vgId = TD_VID(pVnode);
handle->pRef = walOpenRef(pVnode->pWal);
if (handle->pRef == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
TQ_ERR_RETURN(walSetRefVer(handle->pRef, handle->snapshotVer));
@ -213,8 +214,8 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
initStorageAPI(&reader.api);
if (handle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle->execHandle.task =
qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId, &handle->execHandle.numOfCols, handle->consumerId);
handle->execHandle.task = qCreateQueueExecTaskInfo(handle->execHandle.execCol.qmsg, &reader, vgId,
&handle->execHandle.numOfCols, handle->consumerId);
if (handle->execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle->subKey);
return TSDB_CODE_OUT_OF_MEMORY;
@ -251,13 +252,15 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle){
handle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, handle->consumerId);
SArray* tbUidList = NULL;
int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList, handle->execHandle.task);
int ret = qGetTableList(handle->execHandle.execTb.suid, pVnode, handle->execHandle.execTb.node, &tbUidList,
handle->execHandle.task);
if (ret != TDB_CODE_SUCCESS) {
tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle->subKey, handle->consumerId);
taosArrayDestroy(tbUidList);
return TSDB_CODE_SCH_INTERNAL_ERROR;
}
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId,
handle->execHandle.execTb.suid);
handle->execHandle.pTqReader = tqReaderOpen(pVnode);
tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL);
taosArrayDestroy(tbUidList);
@ -304,7 +307,8 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
if (tqMetaInitHandle(pTq, handle) < 0) {
return -1;
}
tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
tqInfo("tqMetaCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey,
handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
}
@ -319,7 +323,8 @@ static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew){
int32_t code = TDB_CODE_SUCCESS;
TQ_ERR_GO_TO_END(tdbTbcOpen(pOld, &pCur, NULL));
TQ_ERR_GO_TO_END(tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(
tdbBegin(pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED));
TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur));
while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) {
@ -454,6 +459,7 @@ int32_t tqMetaTransform(STQ* pTq) {
TQ_ERR_GO_TO_END(tqMetaTransformInfo(pTq->pMetaDB, pCheckStore, pTq->pCheckStore));
TQ_ERR_GO_TO_END(tqBuildFName(&offsetNew, pTq->path, TQ_OFFSET_NAME));
if(taosCheckExistFile(offset)) {
if (taosCopyFile(offset, offsetNew) < 0) {
tqError("copy offset file error");

14
source/dnode/vnode/src/tq/tqRead.c
View File

@ -188,7 +188,7 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t
int64_t committedVer = walGetCommittedVer(pHandle->pWalReader->pWal);
int64_t appliedVer = walGetAppliedVer(pHandle->pWalReader->pWal);
wDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64
tqDebug("vgId:%d, start to fetch wal, index:%" PRId64 ", last:%" PRId64 " commit:%" PRId64 ", applied:%" PRId64
", 0x%" PRIx64,
vgId, offset, lastVer, committedVer, appliedVer, id);
@ -240,9 +240,7 @@ END:
return code;
}
bool tqGetTablePrimaryKey(STqReader* pReader){
return pReader->hasPrimaryKey;
}
bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false;
@ -311,10 +309,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
int32_t code = 0;
while (1) {
code = walNextValidMsg(pReader);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
TAOS_CHECK_RETURN(walNextValidMsg(pReader));
SWalCont* pCont = &pReader->pHead->head;
int64_t ver = pCont->version;
@ -684,7 +679,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
SColData* pCol = taosArrayGet(pCols, sourceIdx);
SColVal colVal;
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual, sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
tqTrace("lostdata colActual:%d, sourceIdx:%d, targetIdx:%d, numOfCols:%d, source cid:%d, dst cid:%d", colActual,
sourceIdx, targetIdx, numOfCols, pCol->cid, pColData->info.colId);
if (pCol->cid < pColData->info.colId) {
sourceIdx++;
} else if (pCol->cid == pColData->info.colId) {

286
source/dnode/vnode/src/tq/tqSink.c
View File

@ -18,8 +18,6 @@
#include "tmsg.h"
#include "tq.h"
#define MAX_CACHE_TABLE_INFO_NUM 10240
typedef struct STableSinkInfo {
uint64_t uid;
tstr name;
@ -43,7 +41,7 @@ static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, con
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
int32_t numOfTags);
static SArray* createDefaultTagColName();
static int32_t createDefaultTagColName(SArray** pList);
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
@ -95,7 +93,10 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
SSingleDeleteReq req = {.startTs = skey, .endTs = ekey};
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1);
taosArrayPush(deleteReq->deleteReqs, &req);
void* p = taosArrayPush(deleteReq->deleteReqs, &req);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (originName) name = originName;
taosMemoryFreeClear(name);
@ -149,14 +150,20 @@ static bool tqGetTableInfo(SSHashObj* pTableInfoMap, uint64_t groupId, STableSin
static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
void* buf = NULL;
int32_t tlen = 0;
encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
int32_t code = encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen);
if (code) {
tqError("vgId:%d failed to encode create table msg, create table failed, code:%s", TD_VID(pVnode), tstrerror(code));
return code;
}
SRpcMsg msg = {.msgType = TDMT_VND_CREATE_TABLE, .pCont = buf, .contLen = tlen};
if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
code = tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg);
if (code) {
tqError("failed to put into write-queue since %s", terrstr());
}
return TSDB_CODE_SUCCESS;
return code;
}
int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
@ -166,18 +173,36 @@ int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const
// set super table name
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
pCreateTableReq->ctb.tagNum = numOfTags;
return TSDB_CODE_SUCCESS;
int32_t code = tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if (code == 0) {
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code
tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName);
}
}
SArray* createDefaultTagColName() {
pCreateTableReq->ctb.tagNum = numOfTags;
return code;
}
int32_t createDefaultTagColName(SArray** pColNameList) {
*pColNameList = NULL;
SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
if (pTagColNameList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
taosArrayPush(pTagColNameList, tagNameStr);
return pTagColNameList;
void* p = taosArrayPush(pTagColNameList, tagNameStr);
if (p == NULL) {
taosArrayDestroy(pTagColNameList);
return TSDB_CODE_OUT_OF_MEMORY;
}
*pColNameList = pTagColNameList;
return TSDB_CODE_SUCCESS;
}
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
@ -204,15 +229,17 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows;
SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId;
tqDebug("s-task:%s build create %d table(s) msg", pTask->id.idStr, rows);
tqDebug("s-task:%s build create %d table(s) msg", id, rows);
int32_t code = 0;
SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) {
tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno));
tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno));
goto _end;
}
@ -222,15 +249,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
if (code) {
tqError("s-task:%s vgId:%d failed to init create table msg", id, vgId);
continue;
}
taosArrayClear(tagArray);
if (size == 2) {
STagVal tagVal = {
.cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(tagArray, &tagVal);
pCreateTbReq->ctb.tagName = createDefaultTagColName();
void* p = taosArrayPush(tagArray, &tagVal);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
if (code) {
return code;
}
} else {
for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
@ -245,14 +283,19 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
} else {
memcpy(&tagVal.i64, pData, pTagData->info.bytes);
}
taosArrayPush(tagArray, &tagVal);
void* p = taosArrayPush(tagArray, &tagVal);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
}
}
tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
code = tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
taosArrayDestroy(tagArray);
tagArray = NULL;
if (pCreateTbReq->ctb.pTag == NULL) {
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
@ -270,22 +313,34 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq);
void* p = taosArrayPush(reqs.pArray, pCreateTbReq);
if (p == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
STableSinkInfo* pInfo = NULL;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo);
if (!alreadyCached) {
code = doCreateSinkInfo(pCreateTbReq->name, &pInfo);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr);
if (code) {
tqError("vgId:%d failed to create sink tableInfo for table:%s, s-task:%s", vgId, pCreateTbReq->name, id);
continue;
}
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, id);
if (code) {
tqError("vgId:%d failed to put sink tableInfo:%s into cache, s-task:%s", vgId, pCreateTbReq->name, id);
}
}
tqDebug("s-task:%s build create table:%s msg complete", id, pCreateTbReq->name);
}
reqs.nReqs = taosArrayGetSize(reqs.pArray);
code = tqPutReqToQueue(pVnode, &reqs);
if (code != TSDB_CODE_SUCCESS) {
tqError("s-task:%s failed to send create table msg", pTask->id.idStr);
tqError("s-task:%s failed to send create table msg", id);
}
_end:
@ -348,15 +403,26 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
SRow* pOldRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k);
if (pNewRow->ts < pOldRow->ts) {
taosArrayPush(pFinal, &pNewRow);
void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1;
} else if (pNewRow->ts > pOldRow->ts) {
taosArrayPush(pFinal, &pOldRow);
void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1;
} else {
// check for the existance of primary key
if (pNewRow->numOfPKs == 0) {
taosArrayPush(pFinal, &pNewRow);
void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1;
j += 1;
tRowDestroy(pOldRow);
@ -369,7 +435,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
int32_t ret = tRowKeyCompare(&kNew, &kOld);
if (ret <= 0) {
taosArrayPush(pFinal, &pNewRow);
void* p = taosArrayPush(pFinal, &pNewRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
j += 1;
if (ret == 0) {
@ -377,7 +447,11 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
tRowDestroy(pOldRow);
}
} else {
taosArrayPush(pFinal, &pOldRow);
void* p = taosArrayPush(pFinal, &pOldRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
k += 1;
}
}
@ -386,12 +460,18 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
while (j < newLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pNew->aRowP, j++);
taosArrayPush(pFinal, &pRow);
void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
while (k < oldLen) {
SRow* pRow = *(SRow**)TARRAY_GET_ELEM(pExisted->aRowP, k++);
taosArrayPush(pFinal, &pRow);
void* p = taosArrayPush(pFinal, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
taosArrayDestroy(pNew->aRowP);
@ -425,34 +505,40 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
return true;
}
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
SSDataBlock* pDataBlock, SArray* pTagArray, bool newSubTableRule) {
int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock,
SArray* pTagArray, bool newSubTableRule, SVCreateTbReq** pReq) {
*pReq = NULL;
SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
if (pCreateTbReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArrayClear(pTagArray);
initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(pTagArray, &tagVal);
tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
int32_t code = initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
if (code != 0) {
return code;
}
pCreateTbReq->ctb.tagName = createDefaultTagColName();
STagVal tagVal = {.cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
void* p = taosArrayPush(pTagArray, &tagVal);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
code = tTagNew(pTagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
if (pCreateTbReq->ctb.pTag == NULL || (code != 0)) {
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
return code;
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
// set table name
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule);
return pCreateTbReq;
*pReq = pCreateTbReq;
return code;
}
int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) {
@ -555,7 +641,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
break;
}
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv);
void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex);
if (colDataIsNull_s(pColData, j)) {
@ -566,7 +655,11 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
}
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type);
taosArrayPush(pVals, &cv);
void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
dataIndex++;
} else {
void* colData = colDataGetData(pColData, j);
@ -574,12 +667,18 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
SValue sv =
(SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv);
void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
} else {
SValue sv = {.type = pCol->type};
memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, sv);
taosArrayPush(pVals, &cv);
void* p = taosArrayPush(pVals, &cv);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
dataIndex++;
}
@ -596,7 +695,10 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
}
ASSERT(pRow);
taosArrayPush(pTableData->aRowP, &pRow);
void* p = taosArrayPush(pTableData->aRowP, &pRow);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
taosArrayDestroy(pVals);
@ -665,6 +767,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t vgId = TD_VID(pVnode);
STableSinkInfo* pTableSinkInfo = NULL;
int32_t code = 0;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, groupId, &pTableSinkInfo);
@ -686,7 +789,11 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
} else { // this groupId has not been kept in cache yet
if (dstTableName[0] == 0) {
memset(dstTableName, 0, TSDB_TABLE_NAME_LEN);
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
code = buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
if (code) {
tqDebug("s-task:%s failed to build auto create table-name:%s, groupId:0x%" PRId64, id, dstTableName, groupId);
return code;
}
} else {
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
@ -699,8 +806,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
}
}
int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
if (code == 0) {
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName);
} else {
tqDebug("s-task:%s failed to build new sinkTableInfo, dstTable:%s", id, dstTableName);
return code;
}
}
if (alreadyCached) {
@ -731,20 +843,20 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq =
code =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
(pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1),
&pTableData->pCreateTbReq);
taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) {
tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName,
tstrerror(terrno));
if (code) {
tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName, tstrerror(code));
taosMemoryFree(pTableSinkInfo);
return terrno;
return code;
}
pTableSinkInfo->uid = 0;
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} else {
metaReaderClear(&mr);
@ -765,12 +877,12 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableSinkInfo->uid = mr.me.uid;
metaReaderClear(&mr);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
code = doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
}
}
}
return TDB_CODE_SUCCESS;
return code;
}
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
@ -864,14 +976,21 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id);
(void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id);
tbData.pCreateTbReq = NULL;
}
continue;
}
taosArrayPush(submitReq.aSubmitTbData, &tbData);
void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqDebug("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
}
code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, 1);
if (code) { // failed and continue
tqDebug("vgId:%d, s-task:%s submit msg failed, data lost", vgId, id);
}
}
}
} else {
@ -918,16 +1037,24 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
if (tbData.pCreateTbReq != NULL) {
tdDestroySVCreateTbReq(tbData.pCreateTbReq);
doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id);
(void) doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id);
tbData.pCreateTbReq = NULL;
}
continue;
}
taosArrayPush(submitReq.aSubmitTbData, &tbData);
void* p = taosArrayPush(submitReq.aSubmitTbData, &tbData);
if (p == NULL) {
tqError("vgId:%d, s-task:%s failed to build submit msg, data lost", vgId, id);
continue;
}
int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
code = taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
if (code) {
tqError("vgId:%d, s-task:%s failed to put group into index map, code:%s", vgId, id, tstrerror(code));
continue;
}
} else {
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id);
if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) {
@ -951,7 +1078,10 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
taosHashCleanup(pTableIndexMap);
if (hasSubmit) {
doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks);
code = doBuildAndSendSubmitMsg(pVnode, pTask, &submitReq, numOfBlocks);
if (code) { // failed and continue
tqError("vgId:%d failed to build and send submit msg", vgId);
}
} else {
tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
tqDebug("vgId:%d, s-task:%s write results completed", vgId, id);
@ -989,7 +1119,11 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char
}
int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId));
if (code == 0) {
tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId);
} else {
tqError("s-task:%s failed to remove table meta from hashmap, groupId:%" PRId64, id, groupId);
}
return code;
}
@ -1019,10 +1153,14 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock*
void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));
tEncoderInit(&encoder, abuf, len);
tEncodeSBatchDeleteReq(&encoder, &deleteReq);
code = tEncodeSBatchDeleteReq(&encoder, &deleteReq);
tEncoderClear(&encoder);
taosArrayDestroy(deleteReq.deleteReqs);
if (code) {
return code;
}
((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode);
SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)};

63
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -37,7 +37,12 @@ int32_t tqScanWal(STQ* pTq) {
// check all tasks
int32_t numOfTasks = 0;
bool shouldIdle = true;
doScanWalForAllTasks(pMeta, &shouldIdle);
int32_t code = doScanWalForAllTasks(pMeta, &shouldIdle);
if (code) {
tqError("vgId:%d failed to start all tasks, try next time", vgId);
return code;
}
streamMetaWLock(pMeta);
int32_t times = (--pMeta->scanInfo.scanCounter);
@ -51,9 +56,13 @@ int32_t tqScanWal(STQ* pTq) {
if (times > 0) {
tqDebug("vgId:%d scan wal for stream tasks for %d times in %dms", vgId, times, SCAN_WAL_IDLE_DURATION);
tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION);
code = tqScanWalInFuture(pTq, numOfTasks, SCAN_WAL_IDLE_DURATION);
if (code) {
tqError("vgId:%d sched scan wal in %dms failed, ignore this failure", vgId, SCAN_WAL_IDLE_DURATION);
}
return 0;
}
return code;
}
typedef struct SBuildScanWalMsgParam {
@ -69,28 +78,44 @@ static void doStartScanWal(void* param, void* tmrId) {
tqDebug("vgId:%d create msg to start wal scan, numOfTasks:%d, vnd restored:%d", vgId, pParam->numOfTasks,
pTq->pVnode->restored);
/*int32_t code = */ streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA);
int32_t code = streamTaskSchedTask(&pTq->pVnode->msgCb, vgId, 0, 0, STREAM_EXEC_T_EXTRACT_WAL_DATA);
taosMemoryFree(pParam);
if (code) {
tqError("vgId:%d failed sched task to scan wal", vgId);
}
}
int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDuration) {
SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t code = 0;
int32_t vgId = TD_VID(pTq->pVnode);
SBuildScanWalMsgParam* pParam = taosMemoryMalloc(sizeof(SBuildScanWalMsgParam));
if (pParam == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pParam->pTq = pTq;
pParam->numOfTasks = numOfTasks;
tmr_h pTimer = streamTimerGetInstance();
ASSERT(pTimer);
tmr_h pTimer = NULL;
code = streamTimerGetInstance(&pTimer);
if (code) {
tqError("vgId:%d failed to get tmr ctrl during sched scan wal", vgId);
return code;
}
if (pMeta->scanInfo.scanTimer == NULL) {
pMeta->scanInfo.scanTimer = taosTmrStart(doStartScanWal, idleDuration, pParam, pTimer);
} else {
taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer);
code = taosTmrReset(doStartScanWal, idleDuration, pParam, pTimer, &pMeta->scanInfo.scanTimer);
if (code) {
tqError("vgId:%d failed to start scan wal in:%dms", vgId, idleDuration);
}
}
return TSDB_CODE_SUCCESS;
return code;
}
int32_t tqScanWalAsync(STQ* pTq, bool ckPause) {
@ -207,7 +232,11 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) {
double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0;
qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs",
id, pTask->step2Range.minVer, maxVer, el);
/*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask);
int32_t code = streamTaskPutTranstateIntoInputQ(pTask);
if (code) {
qError("s-task:%s failed to put trans-state into inputQ", id);
}
return true;
} else {
qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64
@ -290,8 +319,12 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems
break;
}
} else {
walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer);
code = walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer);
if (code) {
tqError("s-task:%s failed to seek ver to:%"PRId64 " in wal", id, pTask->chkInfo.nextProcessVer);
}
break;
}
}
@ -347,18 +380,18 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX;
taosThreadMutexLock(&pTask->lock);
streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask);
if (pState.state != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock);
streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
bool hasNewData = doPutDataIntoInputQ(pTask, maxVer, &numOfItems);
taosThreadMutexUnlock(&pTask->lock);
streamMutexUnlock(&pTask->lock);
if ((numOfItems > 0) || hasNewData) {
noDataInWal = false;
@ -366,7 +399,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
if (code != TSDB_CODE_SUCCESS) {
streamMetaReleaseTask(pStreamMeta, pTask);
taosArrayDestroy(pTaskList);
return -1;
return code;
}
}
@ -379,5 +412,5 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
}
taosArrayDestroy(pTaskList);
return 0;
return TSDB_CODE_SUCCESS;
}

209
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -160,6 +160,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
int64_t st = taosGetTimestampMs();
bool updated = false;
int32_t code = 0;
SStreamTaskNodeUpdateMsg req = {0};
@ -258,26 +259,40 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
}
}
// save
// stream do update the nodeEp info, write it into stream meta.
if (updated) {
tqDebug("s-task:%s vgId:%d save task after update epset, and stop task", idstr, vgId);
streamMetaSaveTask(pMeta, pTask);
code = streamMetaSaveTask(pMeta, pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) {
streamMetaSaveTask(pMeta, *ppHTask);
code = streamMetaSaveTask(pMeta, *ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to save related history task, code:%s", idstr, vgId, tstrerror(code));
}
}
} else {
tqDebug("s-task:%s vgId:%d not save task since not update epset actually, stop task", idstr, vgId);
}
streamTaskStop(pTask);
code = streamTaskStop(pTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop task, code:%s", idstr, vgId, tstrerror(code));
}
if (ppHTask != NULL) {
streamTaskStop(*ppHTask);
code = streamTaskStop(*ppHTask);
if (code) {
tqError("s-task:%s vgId:%d failed to stop related history task, code:%s", idstr, vgId, tstrerror(code));
}
}
// keep info
streamMetaAddIntoUpdateTaskList(pMeta, pTask, (ppHTask != NULL) ? (*ppHTask) : NULL, req.transId, st);
rsp.code = 0;
rsp.code = TSDB_CODE_SUCCESS;
// possibly only handle the stream task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
@ -305,13 +320,16 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
#if 0
taosMSleep(5000);// for test purpose, to trigger the leader election
#endif
tqStreamTaskStartAsync(pMeta, cb, true);
code = tqStreamTaskStartAsync(pMeta, cb, true);
if (code) {
tqError("vgId:%d async start all tasks, failed, code:%s", vgId, tstrerror(code));
}
}
}
streamMetaWUnLock(pMeta);
taosArrayDestroy(req.pNodeList);
return rsp.code;
return rsp.code; // always return true
}
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -333,7 +351,7 @@ int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask) {
if (pTask && (code == 0)) {
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
if (streamProcessDispatchMsg(pTask, &req, &rsp) != 0) {
return -1;
@ -393,14 +411,14 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId, &pTask);
if (pTask) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
if (pTask && (code == 0)) {
code = streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
return code;
} else {
tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, pRsp->upstreamTaskId);
terrno = TSDB_CODE_STREAM_TASK_NOT_EXIST;
return terrno;
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
}
}
@ -408,16 +426,22 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SDecoder decoder;
SStreamRetrieveReq req;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamRetrieveReq(&decoder, &req);
code = tDecodeStreamRetrieveReq(&decoder, &req);
tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d failed to decode retrieve msg, quit handling it", pMeta->vgId);
return code;
}
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask);
if (pTask == NULL) {
code = streamMetaAcquireTask(pMeta, req.streamId, req.dstTaskId, &pTask);
if (pTask == NULL || code != 0) {
tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
req.dstTaskId);
tCleanupStreamRetrieveReq(&req);
@ -446,6 +470,7 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamTaskCheckReq req;
SStreamTaskCheckRsp rsp = {0};
@ -453,9 +478,14 @@ int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen);
tDecodeStreamTaskCheckReq(&decoder, &req);
code = tDecodeStreamTaskCheckReq(&decoder, &req);
tDecoderClear(&decoder);
if (code) {
tqError("vgId:%d decode check msg failed, not handle this msg", pMeta->vgId);
return code;
}
streamTaskProcessCheckMsg(pMeta, &req, &rsp);
return streamTaskSendCheckRsp(pMeta, req.upstreamNodeId, &rsp, &pMsg->info, req.upstreamTaskId);
}
@ -490,7 +520,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, rsp.streamId, rsp.upstreamTaskId, &pTask);
if (pTask == NULL) {
if ((pTask == NULL) || (code != 0)) {
return streamMetaAddFailedTask(pMeta, rsp.streamId, rsp.upstreamTaskId);
}
@ -518,19 +548,25 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId, &pTask);
if (pTask == NULL) {
if (code != 0) {
tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
return code;
}
tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId,
pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
streamMetaReleaseTask(pMeta, pTask);
if (code) {
return code;
}
{ // send checkpoint ready rsp
SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg));
if (pReadyRsp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pReadyRsp->upstreamTaskId = req.upstreamTaskId;
pReadyRsp->upstreamNodeId = req.upstreamNodeId;
@ -606,8 +642,8 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
if (restored) {
SStreamTask* p = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &p);
if ((p != NULL) && (p->info.fillHistory == 0)) {
tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
if ((p != NULL) && (code == 0) && (p->info.fillHistory == 0)) {
code = tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
}
if (p != NULL) {
@ -631,6 +667,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId;
STaskId hTaskId = {0};
tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
@ -649,8 +686,12 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
}
streamTaskSetRemoveBackendFiles(pTask);
streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
code = streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
streamMetaReleaseTask(pMeta, pTask);
if (code) {
tqError("s-task:0x%x failed to clear related fill-history info, still exists", pReq->taskId);
}
}
streamMetaWUnLock(pMeta);
@ -658,11 +699,17 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
// drop the related fill-history task firstly
if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
code = streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x failed", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
}
}
// drop the stream task now
streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
code = streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
if (code) {
tqDebug("s-task:0x%x vgId:%d drop task failed", pReq->taskId, vgId);
}
// commit the update
streamMetaWLock(pMeta);
@ -674,12 +721,13 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
}
streamMetaWUnLock(pMeta);
return 0;
return 0; // always return success
}
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored, char* msg) {
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
int32_t code = 0;
int32_t vgId = pMeta->vgId;
tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId);
@ -689,7 +737,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask != NULL && (*ppTask) != NULL) {
streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq);
code = streamTaskUpdateTaskCheckpointInfo(*ppTask, restored, pReq);
} else { // failed to get the task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
tqError(
@ -700,7 +748,7 @@ int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, bool restored
streamMetaWUnLock(pMeta);
// always return success when handling the requirement issued by mnode during transaction.
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
@ -746,7 +794,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
if (isLeader && !tsDisableStream) {
streamMetaWUnLock(pMeta);
streamMetaStartAllTasks(pMeta);
code = streamMetaStartAllTasks(pMeta);
} else {
streamMetaResetStartInfo(&pMeta->startInfo, pMeta->vgId);
pMeta->startInfo.restartCount = 0;
@ -765,16 +813,16 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
int32_t vgId = pMeta->vgId;
if (type == STREAM_EXEC_T_START_ONE_TASK) {
streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
(void) streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId);
return 0;
} else if (type == STREAM_EXEC_T_START_ALL_TASKS) {
streamMetaStartAllTasks(pMeta);
(void) streamMetaStartAllTasks(pMeta);
return 0;
} else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) {
restartStreamTasks(pMeta, isLeader);
(void) restartStreamTasks(pMeta, isLeader);
return 0;
} else if (type == STREAM_EXEC_T_STOP_ALL_TASKS) {
streamMetaStopAllTasks(pMeta);
(void) streamMetaStopAllTasks(pMeta);
return 0;
} else if (type == STREAM_EXEC_T_ADD_FAILED_TASK) {
int32_t code = streamMetaAddFailedTask(pMeta, pReq->streamId, pReq->taskId);
@ -783,7 +831,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) {
if (pTask != NULL && (code == 0)) {
char* pStatus = NULL;
if (streamTaskReadyToRun(pTask, &pStatus)) {
int64_t execTs = pTask->status.lastExecTs;
@ -804,12 +852,12 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
if ((pTask != NULL) && (code == 0)) { // even in halt status, the data in inputQ must be processed
char* p = NULL;
if (streamTaskReadyToRun(pTask, &p)) {
tqDebug("vgId:%d s-task:%s status:%s start to process block from inputQ, next checked ver:%" PRId64, vgId,
pTask->id.idStr, p, pTask->chkInfo.nextProcessVer);
streamExecTask(pTask);
(void) streamExecTask(pTask);
} else {
int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
@ -829,6 +877,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
STaskStartInfo* pStartInfo = &pMeta->startInfo;
int32_t vgId = pMeta->vgId;
bool scanWal = false;
int32_t code = 0;
streamMetaWLock(pMeta);
if (pStartInfo->startAllTasks == 1) {
@ -844,8 +893,7 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
pStartInfo->restartCount);
streamMetaWUnLock(pMeta);
restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER));
return TSDB_CODE_SUCCESS;
return restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER));
} else {
if (pStartInfo->restartCount == 0) {
tqDebug("vgId:%d start all tasks completed in callbackFn, restartCount is 0", pMeta->vgId);
@ -862,10 +910,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) {
if (scanWal && (vgId != SNODE_HANDLE)) {
tqDebug("vgId:%d start scan wal for executing tasks", vgId);
tqScanWalAsync(pMeta->ahandle, true);
code = tqScanWalAsync(pMeta->ahandle, true);
}
return TSDB_CODE_SUCCESS;
return code;
}
int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
@ -873,7 +921,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->taskId);
return TSDB_CODE_SUCCESS;
@ -881,7 +929,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
taosThreadMutexLock(&pTask->lock);
streamMutexLock(&pTask->lock);
streamTaskClearCheckInfo(pTask, true);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks
@ -904,7 +952,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, char* pMsg) {
tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState.name);
}
taosThreadMutexUnlock(&pTask->lock);
streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
@ -915,7 +963,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64
" from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId);
@ -929,11 +977,10 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready",
pTask->id.idStr, (int32_t)pReq->downstreamTaskId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_STREAM_TASK_IVLD_STATUS);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
return code;
}
SStreamTaskState pState = streamTaskGetStatus(pTask);
@ -948,7 +995,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
// re-send the lost checkpoint-trigger msg to downstream task
tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr,
(int32_t)pReq->downstreamTaskId, checkpointId, transId);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_SUCCESS);
} else { // not send checkpoint-trigger yet, wait
int32_t recv = 0, total = 0;
@ -962,7 +1009,7 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"sending checkpoint-source/trigger",
pTask->id.idStr, recv, total);
}
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
}
} else { // upstream not recv the checkpoint-source/trigger till now
@ -971,12 +1018,12 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg)
"s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all "
"upstream sending checkpoint-source/trigger",
pTask->id.idStr);
streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
code = streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info,
TSDB_CODE_ACTION_IN_PROGRESS);
}
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -984,19 +1031,19 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg)
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError(
"vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, pRsp->taskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
return code;
}
tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d",
pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId);
streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
code = streamTaskProcessCheckpointTriggerRsp(pTask, pRsp);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
@ -1004,7 +1051,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
pReq->taskId);
// since task is in [STOP|DROPPING] state, it is safe to assume the pause is active
@ -1018,7 +1065,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) {
if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pHistoryTask = NULL;
code = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId, &pHistoryTask);
if (pHistoryTask == NULL) {
if (pHistoryTask == NULL || (code != 0)) {
tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
", it may have been dropped already",
pMeta->vgId, pTask->hTaskInfo.id.taskId);
@ -1042,6 +1089,8 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
bool fromVnode) {
SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle;
int32_t vgId = pMeta->vgId;
int32_t code = 0;
if (pTask == NULL) {
return -1;
}
@ -1065,18 +1114,18 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
pTask->hTaskInfo.operatorOpen = false;
streamStartScanHistoryAsync(pTask, igUntreated);
code = streamStartScanHistoryAsync(pTask, igUntreated);
} else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
tqScanWalAsync((STQ*)handle, false);
code = tqScanWalAsync((STQ*)handle, false);
} else {
streamTrySchedExec(pTask);
code = streamTrySchedExec(pTask);
}
} /*else {
ASSERT(status != TASK_STATUS__UNINIT);
}*/
streamMetaReleaseTask(pMeta, pTask);
return 0;
return code;
}
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) {
@ -1086,15 +1135,15 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError("s-task:0x%x failed to acquire task to resume, it may have been dropped or stopped", pReq->taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
}
taosThreadMutexLock(&pTask->lock);
streamMutexLock(&pTask->lock);
SStreamTaskState pState = streamTaskGetStatus(pTask);
tqDebug("s-task:%s start to resume from paused, current status:%s", pTask->id.idStr, pState.name);
taosThreadMutexUnlock(&pTask->lock);
streamMutexUnlock(&pTask->lock);
code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode);
if (code != 0) {
@ -1104,11 +1153,11 @@ int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* m
STaskId* pHTaskId = &pTask->hTaskInfo.id;
SStreamTask* pHTask = NULL;
code = streamMetaAcquireTask(pMeta, pHTaskId->streamId, pHTaskId->taskId, &pHTask);
if (pHTask) {
taosThreadMutexLock(&pHTask->lock);
if (pHTask && (code == 0)) {
streamMutexLock(&pHTask->lock);
SStreamTaskState p = streamTaskGetStatus(pHTask);
tqDebug("s-task:%s related history task start to resume from paused, current status:%s", pHTask->id.idStr, p.name);
taosThreadMutexUnlock(&pHTask->lock);
streamMutexUnlock(&pHTask->lock);
code = tqProcessTaskResumeImpl(handle, pHTask, sversion, pReq->igUntreated, fromVnode);
}
@ -1139,15 +1188,15 @@ int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
int32_t code = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped",
pRsp->downstreamNodeId, pRsp->downstreamTaskId);
return TSDB_CODE_STREAM_TASK_NOT_EXIST;
return code;
}
streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId);
code = streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -1173,11 +1222,11 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, req.streamId, req.taskId, &pTask);
if (pTask == NULL) {
if (pTask == NULL || (code != 0)) {
tqError("vgId:%d process set consensus checkpointId req, failed to acquire task:0x%x, it may have been dropped already",
pMeta->vgId, req.taskId);
streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return TSDB_CODE_SUCCESS;
(void)streamMetaAddFailedTask(pMeta, req.streamId, req.taskId);
return code;
}
// discard the rsp, since it is expired.
@ -1193,13 +1242,13 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
tqDebug("s-task:%s vgId:%d checkpointId:%" PRId64 " restore to consensus-checkpointId:%" PRId64 " from mnode",
pTask->id.idStr, vgId, pTask->chkInfo.checkpointId, req.checkpointId);
taosThreadMutexLock(&pTask->lock);
streamMutexLock(&pTask->lock);
ASSERT(pTask->chkInfo.checkpointId >= req.checkpointId);
if (pTask->chkInfo.consensusTransId >= req.transId) {
tqDebug("s-task:%s vgId:%d latest consensus transId:%d, expired consensus trans:%d, discard",
pTask->id.idStr, vgId, pTask->chkInfo.consensusTransId, req.transId);
taosThreadMutexUnlock(&pTask->lock);
streamMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
}
@ -1215,14 +1264,14 @@ int32_t tqStreamTaskProcessConsenChkptIdReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
}
pTask->chkInfo.consensusTransId = req.transId;
taosThreadMutexUnlock(&pTask->lock);
streamMutexUnlock(&pTask->lock);
if (pMeta->role == NODE_ROLE_LEADER) {
/*code = */ tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId);
code = tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, req.streamId, req.taskId);
} else {
tqDebug("vgId:%d follower not start task:%s", vgId, pTask->id.idStr);
}
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS;
return code;
}

269
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -23,20 +23,27 @@
#define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) {
static int32_t setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) {
char* buf = taosMemoryCalloc(1, pCol->info.bytes);
if (buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE);
pRes->bytes = 0;
pRes->hasResult = true;
pRes->isNull = true;
varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE);
colDataSetVal(pCol, row, buf, false);
int32_t code = colDataSetVal(pCol, row, buf, false);
taosMemoryFree(buf);
return code;
}
static void saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, const int32_t slotId,
static int32_t saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, const int32_t slotId,
SColumnInfoData* pColInfoData, int32_t numOfRows) {
SColVal* pVal = &pColVal->colVal;
int32_t code = 0;
// allNullRow = false;
if (IS_VAR_DATA_TYPE(pColVal->colVal.value.type)) {
@ -46,17 +53,19 @@ static void saveOneRowForLastRaw(SLastCol* pColVal, SCacheRowsReader* pReader, c
varDataSetLen(pReader->transferBuf[slotId], pVal->value.nData);
memcpy(varDataVal(pReader->transferBuf[slotId]), pVal->value.pData, pVal->value.nData);
colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false);
code = colDataSetVal(pColInfoData, numOfRows, pReader->transferBuf[slotId], false);
}
} else {
colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal));
code = colDataSetVal(pColInfoData, numOfRows, (const char*)&pVal->value.val, !COL_VAL_IS_VALUE(pVal));
}
return;
return code;
}
static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds,
const int32_t* dstSlotIds, void** pRes, const char* idStr) {
int32_t numOfRows = pBlock->info.rows;
int32_t code = 0;
if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
uint64_t ts = TSKEY_MIN;
@ -64,14 +73,33 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
col_id_t colId = -1;
SArray* funcTypeBlockArray = taosArrayInit(pReader->numOfCols, sizeof(int32_t));
if (funcTypeBlockArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t i = 0; i < pReader->numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
int32_t funcType = FUNCTION_TYPE_CACHE_LAST;
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t funcType = FUNCTION_TYPE_CACHE_LAST;
if (pReader->pFuncTypeList != NULL && taosArrayGetSize(pReader->pFuncTypeList) > i) {
funcType = *(int32_t*)taosArrayGet(pReader->pFuncTypeList, i);
taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], taosArrayGet(pReader->pFuncTypeList, i));
void* pVal = taosArrayGet(pReader->pFuncTypeList, i);
if (pVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
funcType = *(int32_t*) pVal;
pVal = taosArrayGet(pReader->pFuncTypeList, i);
if (pVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
void* px = taosArrayInsert(funcTypeBlockArray, dstSlotIds[i], pVal);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (slotIds[i] == -1) {
@ -79,15 +107,27 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
colDataSetNULL(pColInfoData, numOfRows);
continue;
}
setFirstLastResColToNull(pColInfoData, numOfRows);
code = setFirstLastResColToNull(pColInfoData, numOfRows);
if (code) {
return code;
}
continue;
}
int32_t slotId = slotIds[i];
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
colId = pColVal->colVal.cid;
if (pColVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
colId = pColVal->colVal.cid;
if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) {
saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows);
code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows);
if (code) {
return code;
}
continue;
}
@ -112,13 +152,25 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
// pColInfoData->info.bytes includes the VARSTR_HEADER_SIZE, need to subtract it
p->hasResult = true;
varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE);
colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false);
code = colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false);
if (code) {
return code;
}
}
for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
if (pCol == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (idx < funcTypeBlockArray->size) {
int32_t funcType = *(int32_t*)taosArrayGet(funcTypeBlockArray, idx);
void* pVal = taosArrayGet(funcTypeBlockArray, idx);
if (pVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t funcType = *(int32_t*)pVal;
if (FUNCTION_TYPE_CACHE_LAST_ROW == funcType) {
continue;
}
@ -128,12 +180,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
if (ts == TSKEY_MIN) {
colDataSetNULL(pCol, numOfRows);
} else {
colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
code = colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
if (code) {
return code;
}
}
continue;
} else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) {
if (p && !p->isNull) {
colDataSetVal(pCol, numOfRows, p->buf, false);
code = colDataSetVal(pCol, numOfRows, p->buf, false);
if (code) {
return code;
}
} else {
colDataSetNULL(pCol, numOfRows);
}
@ -146,15 +204,25 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
} else if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST_ROW)) {
for (int32_t i = 0; i < pReader->numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
int32_t slotId = slotIds[i];
if (slotId == -1) {
colDataSetNULL(pColInfoData, numOfRows);
continue;
}
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows);
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
if (pColVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
code = saveOneRowForLastRaw(pColVal, pReader, slotId, pColInfoData, numOfRows);
if (code) {
return code;
}
}
// pBlock->info.rows += allNullRow ? 0 : 1;
@ -164,7 +232,7 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
return code;
}
static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* idstr) {
@ -206,7 +274,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS;
return (pReader->pLDataIterArray != NULL) ? TSDB_CODE_SUCCESS : TSDB_CODE_OUT_OF_MEMORY;
}
int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols,
@ -269,18 +337,22 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
}
p->idstr = taosStrdup(idstr);
taosThreadMutexInit(&p->readerMutex, NULL);
code = taosThreadMutexInit(&p->readerMutex, NULL);
if (code) {
tsdbCacherowsReaderClose(p);
return code;
}
p->lastTs = INT64_MIN;
*pReader = p;
return TSDB_CODE_SUCCESS;
return code;
}
void* tsdbCacherowsReaderClose(void* pReader) {
void tsdbCacherowsReaderClose(void* pReader) {
SCacheRowsReader* p = pReader;
if (p == NULL) {
return NULL;
return;
}
if (p->pSchema != NULL) {
@ -294,17 +366,25 @@ void* tsdbCacherowsReaderClose(void* pReader) {
taosMemoryFree(p->pCurrSchema);
if (p->rowKey.numOfPKs > 0) {
for (int32_t i = 0; i < p->rowKey.numOfPKs; i++) {
if (IS_VAR_DATA_TYPE(p->rowKey.pks[i].type)) {
taosMemoryFree(p->rowKey.pks[i].pData);
}
}
}
if (p->pLDataIterArray) {
destroySttBlockReader(p->pLDataIterArray, NULL);
}
if (p->pFileReader) {
tsdbDataFileReaderClose(&p->pFileReader);
(void) tsdbDataFileReaderClose(&p->pFileReader);
p->pFileReader = NULL;
}
taosMemoryFree((void*)p->idstr);
taosThreadMutexDestroy(&p->readerMutex);
(void) taosThreadMutexDestroy(&p->readerMutex);
if (p->pTableMap) {
void* pe = NULL;
@ -322,16 +402,28 @@ void* tsdbCacherowsReaderClose(void* pReader) {
}
taosMemoryFree(pReader);
return NULL;
}
static void freeItem(void* pItem) {
static void freeItemOfRow(void* pItem) {
SLastCol* pCol = (SLastCol*)pItem;
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData);
}
}
static void freeItemWithPk(void* pItem) {
SLastCol* pCol = (SLastCol*)pItem;
for (int i = 0; i < pCol->rowKey.numOfPKs; i++) {
if (IS_VAR_DATA_TYPE(pCol->rowKey.pks[i].type)) {
taosMemoryFree(pCol->rowKey.pks[i].pData);
}
}
if (IS_VAR_DATA_TYPE(pCol->colVal.value.type) && pCol->colVal.value.pData) {
taosMemoryFree(pCol->colVal.value.pData);
}
}
static int32_t tsdbCacheQueryReseek(void* pQHandle) {
int32_t code = 0;
SCacheRowsReader* pReader = pQHandle;
@ -342,8 +434,7 @@ static int32_t tsdbCacheQueryReseek(void* pQHandle) {
// just wait for the big all tables' snapshot untaking for now
code = TSDB_CODE_VND_QUERY_BUSY;
taosThreadMutexUnlock(&pReader->readerMutex);
(void)taosThreadMutexUnlock(&pReader->readerMutex);
return code;
} else if (code == EBUSY) {
@ -360,9 +451,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
SCacheRowsReader* pr = pReader;
int32_t code = TSDB_CODE_SUCCESS;
SArray* pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol));
bool hasRes = false;
SArray* pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol));
if (pRow == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
void** pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES);
if (pRes == NULL) {
@ -375,11 +471,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes;
pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + pkBufLen + VARSTR_HEADER_SIZE);
if (pRes[j] == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
p->ts = INT64_MIN;
}
taosThreadMutexLock(&pr->readerMutex);
(void)taosThreadMutexLock(&pr->readerMutex);
code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap);
if (code != TSDB_CODE_SUCCESS) {
goto _end;
@ -401,16 +502,44 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
int32_t slotId = slotIds[i];
if (slotId == -1) {
SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL};
taosArrayPush(pLastCols, &p);
void* px = taosArrayPush(pLastCols, &p);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
continue;
}
struct STColumn* pCol = &pr->pSchema->columns[slotId];
SLastCol p = {.rowKey.ts = INT64_MIN, .colVal.value.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
if (pr->rowKey.numOfPKs > 0) {
p.rowKey.numOfPKs = pr->rowKey.numOfPKs;
for (int32_t j = 0; j < pr->rowKey.numOfPKs; j++) {
p.rowKey.pks[j].type = pr->pkColumn.type;
if (IS_VAR_DATA_TYPE(pr->pkColumn.type)) {
p.rowKey.pks[j].pData = taosMemoryCalloc(1, pr->pkColumn.bytes);
if (p.rowKey.pks[j].pData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
}
}
}
if (IS_VAR_DATA_TYPE(pCol->type)) {
p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
if (p.colVal.value.pData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
}
void* px = taosArrayPush(pLastCols, &p);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
taosArrayPush(pLastCols, &p);
}
int64_t st = taosGetTimestampUs();
@ -418,9 +547,15 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
for (int32_t i = 0; i < pr->numOfTables; ++i) {
tb_uid_t uid = pTableList[i].uid;
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if (code == -1) {// fix the invalid return code
code = 0;
} else if (code != 0) {
goto _end;
}
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, freeItem);
taosArrayClearEx(pRow, freeItemOfRow);
continue;
}
@ -430,9 +565,16 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
for (int32_t k = 0; k < pr->numOfCols; ++k) {
if (slotIds[k] == -1) continue;
SLastCol* p = taosArrayGet(pLastCols, k);
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (pColVal->rowKey.ts > p->rowKey.ts) {
SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
if (pColVal == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (tRowKeyCompare(&pColVal->rowKey, &p->rowKey) > 0) {
if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) {
if (!COL_VAL_IS_VALUE(&p->colVal)) {
hasNotNullRow = false;
@ -445,9 +587,22 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
hasRes = true;
p->rowKey.ts = pColVal->rowKey.ts;
for (int32_t j = 0; j < p->rowKey.numOfPKs; j++) {
if (IS_VAR_DATA_TYPE(p->rowKey.pks[j].type)) {
memcpy(p->rowKey.pks[j].pData, pColVal->rowKey.pks[j].pData, pColVal->rowKey.pks[j].nData);
p->rowKey.pks[j].nData = pColVal->rowKey.pks[j].nData;
} else {
p->rowKey.pks[j].val = pColVal->rowKey.pks[j].val;
}
}
if (k == 0) {
if (TARRAY_SIZE(pTableUidList) == 0) {
taosArrayPush(pTableUidList, &uid);
void* px = taosArrayPush(pTableUidList, &uid);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
} else {
taosArraySet(pTableUidList, 0, &uid);
}
@ -483,28 +638,46 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
}
}
taosArrayClearEx(pRow, freeItem);
taosArrayClearEx(pRow, freeItemOfRow);
}
if (hasRes) {
saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
code = saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
if (code) {
goto _end;
}
}
taosArrayDestroyEx(pLastCols, freeItem);
taosArrayDestroyEx(pLastCols, freeItemWithPk);
} else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) {
tb_uid_t uid = pTableList[i].uid;
tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
if ((code = tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype)) != 0) {
if (code == -1) {// fix the invalid return code
code = 0;
} else if (code != 0) {
goto _end;
}
}
if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
taosArrayClearEx(pRow, freeItem);
taosArrayClearEx(pRow, freeItemOfRow);
continue;
}
saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
taosArrayClearEx(pRow, freeItem);
code = saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
if (code) {
goto _end;
}
taosArrayPush(pTableUidList, &uid);
taosArrayClearEx(pRow, freeItemOfRow);
void* px = taosArrayPush(pTableUidList, &uid);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
++pr->tableIndex;
if (pResBlock->info.rows >= pResBlock->info.capacity) {
@ -521,7 +694,7 @@ _end:
pr->pCurFileSet = NULL;
}
taosThreadMutexUnlock(&pr->readerMutex);
(void)taosThreadMutexUnlock(&pr->readerMutex);
if (pRes != NULL) {
for (int32_t j = 0; j < pr->numOfCols; ++j) {

12
source/dnode/vnode/src/tsdb/tsdbFSet2.c
View File

@ -18,14 +18,14 @@
int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl) {
if (!(lvl[0] = taosMemoryMalloc(sizeof(SSttLvl)))) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
lvl[0]->level = level;
TARRAY2_INIT(lvl[0]->fobjArr);
return 0;
}
static void tsdbSttLvlClearFObj(void *data) { tsdbTFileObjUnref(*(STFileObj **)data); }
static void tsdbSttLvlClearFObj(void *data) { TAOS_UNUSED(tsdbTFileObjUnref(*(STFileObj **)data)); }
int32_t tsdbSttLvlClear(SSttLvl **lvl) {
if (lvl[0] != NULL) {
@ -451,7 +451,9 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f
int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
fset[0] = taosMemoryCalloc(1, sizeof(STFileSet));
if (fset[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (fset[0] == NULL) {
return terrno;
}
fset[0]->fid = fid;
fset[0]->maxVerValid = VERSION_MAX;
@ -543,7 +545,9 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_
int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever,
STFileSetRange **fsr) {
fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0]));
if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
if (fsr[0] == NULL) {
return terrno;
}
fsr[0]->fid = fset1->fid;
fsr[0]->sver = sver;
fsr[0]->ever = ever;

2
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -227,7 +227,7 @@ int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));
if (!fobj[0]) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
taosThreadMutexInit(&fobj[0]->mutex, NULL);

4
source/dnode/vnode/src/tsdb/tsdbIter.c
View File

@ -507,7 +507,7 @@ int32_t tsdbIterOpen(const STsdbIterConfig *config, STsdbIter **iter) {
iter[0] = taosMemoryCalloc(1, sizeof(*iter[0]));
if (iter[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
iter[0]->type = config->type;
@ -677,7 +677,7 @@ int32_t tsdbIterMergerOpen(const TTsdbIterArray *iterArray, SIterMerger **merger
merger[0] = taosMemoryCalloc(1, sizeof(*merger[0]));
if (merger[0] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
merger[0]->isTomb = isTomb;

47
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -54,7 +54,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
if (pMemTable == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _err;
}
taosInitRWLatch(&pMemTable->latch);
@ -71,7 +71,7 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
pMemTable->nBucket = MEM_MIN_HASH;
pMemTable->aBucket = (STbData **)taosMemoryCalloc(pMemTable->nBucket, sizeof(STbData *));
if (pMemTable->aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
taosMemoryFree(pMemTable);
goto _err;
}
@ -174,7 +174,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
// do delete
SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData));
if (pDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _err;
}
pDelData->version = version;
@ -195,7 +195,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
pMemTable->minVer = TMIN(pMemTable->minVer, version);
pMemTable->maxVer = TMAX(pMemTable->maxVer, version);
tsdbCacheDel(pTsdb, suid, uid, sKey, eKey);
TAOS_UNUSED(tsdbCacheDel(pTsdb, suid, uid, sKey, eKey));
tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
" at version %" PRId64,
@ -214,7 +214,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backwa
(*ppIter) = (STbDataIter *)taosMemoryCalloc(1, sizeof(STbDataIter));
if ((*ppIter) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -329,7 +329,7 @@ static int32_t tsdbMemTableRehash(SMemTable *pMemTable) {
int32_t nBucket = pMemTable->nBucket * 2;
STbData **aBucket = (STbData **)taosMemoryCalloc(nBucket, sizeof(STbData *));
if (aBucket == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -369,7 +369,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
ASSERT(pPool != NULL);
pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
if (pTbData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
pTbData->suid = suid;
@ -520,7 +520,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN
ASSERT(0);
}
if (pNode == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -589,7 +589,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
// copy and construct block data
SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData));
if (pBlockData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -599,7 +599,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
pBlockData->aUid = NULL;
pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData);
if (pBlockData->aVersion == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
for (int32_t i = 0; i < pBlockData->nRow; i++) { // todo: here can be optimized
@ -608,7 +608,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData);
if (pBlockData->aTSKEY == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData);
@ -616,7 +616,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
pBlockData->nColData = nColData - 1;
pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData);
if (pBlockData->aColData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -661,7 +661,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
}
if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData);
TAOS_UNUSED(tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData));
}
// SMemTable
@ -720,7 +720,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
pTbData->maxKey = key.key.ts;
}
if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow);
TAOS_UNUSED(tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow));
}
// SMemTable
@ -780,22 +780,3 @@ static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) {
return 0;
}
SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
SArray *aTbDataP = taosArrayInit(pMemTable->nTbData, sizeof(STbData *));
if (aTbDataP == NULL) goto _exit;
for (int32_t iBucket = 0; iBucket < pMemTable->nBucket; iBucket++) {
STbData *pTbData = pMemTable->aBucket[iBucket];
while (pTbData) {
taosArrayPush(aTbDataP, &pTbData);
pTbData = pTbData->next;
}
}
taosArraySort(aTbDataP, tbDataPCmprFn);
_exit:
return aTbDataP;
}

197
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -48,7 +48,7 @@ typedef struct {
static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo);
static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
STsdbReader* pReader);
static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRow);
static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes);
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
STsdbReader* pReader);
static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
@ -73,8 +73,8 @@ static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBloc
STsdbReader* pReader);
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost);
static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
int8_t* pLevel);
static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
int8_t* pLevel, STsdb** pTsdb);
static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level);
static int32_t doBuildDataBlock(STsdbReader* pReader);
static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader);
@ -123,7 +123,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_
pKey->pks[0].val = cv.value.val;
} else {
pKey->pks[0].nData = cv.value.nData;
(void)(void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData);
(void)memcpy(pKey->pks[0].pData, cv.value.pData, cv.value.nData);
}
}
@ -439,7 +439,6 @@ static int32_t tsdbUninitReaderLock(STsdbReader* pReader) {
tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
code = taosThreadMutexDestroy(&pReader->readerMutex);
tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
return code;
@ -450,7 +449,6 @@ static int32_t tsdbAcquireReader(STsdbReader* pReader) {
tsdbTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
code = taosThreadMutexLock(&pReader->readerMutex);
tsdbTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
return code;
@ -541,8 +539,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void
}
initReaderStatus(&pReader->status);
pReader->pTsdb = getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level);
getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level, &pReader->pTsdb);
pReader->info.suid = pCond->suid;
pReader->info.order = pCond->order;
@ -550,6 +547,11 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void
pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows);
pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL;
if (idstr != NULL && pReader->idStr == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pReader->type = pCond->type;
pReader->bFilesetDelimited = false;
pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket
@ -586,6 +588,11 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void
}
pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]);
if (pReader->status.pPrimaryTsCol == NULL) {
code = TSDB_CODE_INVALID_PARA;
goto _end;
}
int32_t type = pReader->status.pPrimaryTsCol->info.type;
if (type != TSDB_DATA_TYPE_TIMESTAMP) {
tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name,
@ -651,7 +658,6 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileRead
STableUidList* pList = &pReader->status.uidList;
int32_t i = 0;
while (i < TARRAY2_SIZE(pBlkArray)) {
pBrinBlk = &pBlkArray->data[i];
if (pBrinBlk->maxTbid.suid < pReader->info.suid) {
@ -824,6 +830,10 @@ static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, S
p1 = taosArrayPush(pTableScanInfoList, &pScanInfo);
} else {
STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if ((*p)->uid != uid) {
p1 = taosArrayPush(pTableScanInfoList, &pScanInfo);
}
@ -1273,6 +1283,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
int32_t rowIndex = 0;
SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
i += 1;
@ -1288,6 +1302,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
colIndex += 1;
} else if (pData->cid == pSupInfo->colId[i]) {
pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
colDataSetNNULL(pColData, 0, dumpedRows);
@ -1309,6 +1326,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
i += 1;
} else { // the specified column does not exist in file block, fill with null data
pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
colDataSetNNULL(pColData, 0, dumpedRows);
i += 1;
}
@ -1317,6 +1338,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
// fill the mis-matched columns with null value
while (i < numOfOutputCols) {
pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
colDataSetNNULL(pColData, 0, dumpedRows);
i += 1;
}
@ -1448,7 +1473,15 @@ static bool getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockIn
}
STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);
if (pTableDataBlockIdx == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
blockInfoToRecord(pRecord, p, pSupInfo);
*nextIndex = pBlockInfo->tbBlockIdx + step;
@ -1462,13 +1495,21 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
return TSDB_CODE_FAILED;
}
SFileDataBlockInfo fblock = *(SFileDataBlockInfo*)taosArrayGet(pBlockIter->blockList, index);
void* p = taosArrayGet(pBlockIter->blockList, index);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SFileDataBlockInfo fblock = *(SFileDataBlockInfo*) p;
pBlockIter->index += step;
if (index != pBlockIter->index) {
if (index > pBlockIter->index) {
for (int32_t i = index - 1; i >= pBlockIter->index; --i) {
SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i);
if (pBlockInfo == NULL) {
return TSDB_CODE_INVALID_PARA;
}
STableBlockScanInfo* pBlockScanInfo = NULL;
code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);
@ -1477,6 +1518,10 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
}
STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);
if (pTableDataBlockIdx == NULL) {
return TSDB_CODE_INVALID_PARA;
}
pTableDataBlockIdx->globalIndex = i + 1;
taosArraySet(pBlockIter->blockList, i + 1, pBlockInfo);
@ -1484,6 +1529,9 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
} else if (index < pBlockIter->index) {
for (int32_t i = index + 1; i <= pBlockIter->index; ++i) {
SFileDataBlockInfo* pBlockInfo = taosArrayGet(pBlockIter->blockList, i);
if (pBlockInfo == NULL) {
return TSDB_CODE_INVALID_PARA;
}
STableBlockScanInfo* pBlockScanInfo = NULL;
code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);
@ -1492,8 +1540,11 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
}
STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);
pTableDataBlockIdx->globalIndex = i - 1;
if (pTableDataBlockIdx == NULL) {
return TSDB_CODE_INVALID_PARA;
}
pTableDataBlockIdx->globalIndex = i - 1;
taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo);
}
}
@ -1506,6 +1557,10 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
}
STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx);
if (pTableDataBlockIdx == NULL) {
return TSDB_CODE_INVALID_PARA;
}
pTableDataBlockIdx->globalIndex = pBlockIter->index;
}
@ -2338,7 +2393,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
pSttBlockReader->uid = pScanInfo->uid;
// second time init stt block reader
if (pScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) {
if (pScanInfo->cleanSttBlocks && (pReader->info.execMode == READER_EXEC_ROWS)) {
return !pScanInfo->sttBlockReturned;
}
@ -2374,6 +2429,9 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
};
SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))};
if (info.pKeyRangeList == NULL) {
return false;
}
int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
if (code != TSDB_CODE_SUCCESS) {
@ -2398,6 +2456,10 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
// calculate the time window for data in stt files
for (int32_t i = 0; i < taosArrayGetSize(info.pKeyRangeList); ++i) {
SSttKeyRange* pKeyRange = taosArrayGet(info.pKeyRangeList, i);
if (pKeyRange == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (pkCompEx(&pScanInfo->sttRange.skey, &pKeyRange->skey) > 0) {
tRowKeyAssign(&pScanInfo->sttRange.skey, &pKeyRange->skey);
}
@ -2618,6 +2680,9 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock
// 1. find the next neighbor block in the scan block list
STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex);
if (tableDataBlockIdx == NULL) {
return TSDB_CODE_INVALID_PARA;
}
// 2. remove it from the scan block list
int32_t neighborIndex = tableDataBlockIdx->globalIndex;
@ -2787,6 +2852,9 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t orde
taosArrayClear(pBlockScanInfo->delSkyline);
} else {
pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY));
if (pBlockScanInfo->delSkyline == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
SArray* pSource = pBlockScanInfo->pFileDelData;
@ -3412,6 +3480,9 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter)
static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
SBlockNumber num = {0};
SArray* pTableList = taosArrayInit(40, POINTER_BYTES);
if (pTableList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t code = moveToNextFile(pReader, &num, pTableList);
if (code != TSDB_CODE_SUCCESS) {
@ -3563,8 +3634,13 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
}
}
static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr,
int8_t* pLevel) {
static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr,
int8_t* pLevel, STsdb** pTsdb) {
if (pTsdb == NULL) {
return;
}
*pTsdb = NULL;
if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) {
int8_t level = 0;
int8_t precision = pVnode->config.tsdbCfg.precision;
@ -3573,7 +3649,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SR
: (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
: 1000000L);
for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
SRetention* pRetention = retentions + level;
if (pRetention->keep <= 0) {
if (level > 0) {
@ -3592,19 +3668,22 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SR
if (level == TSDB_RETENTION_L0) {
*pLevel = TSDB_RETENTION_L0;
tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str);
return VND_RSMA0(pVnode);
*pTsdb = VND_RSMA0(pVnode);
return;
} else if (level == TSDB_RETENTION_L1) {
*pLevel = TSDB_RETENTION_L1;
tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str);
return VND_RSMA1(pVnode);
*pTsdb = VND_RSMA1(pVnode);
return;
} else {
*pLevel = TSDB_RETENTION_L2;
tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str);
return VND_RSMA2(pVnode);
*pTsdb = VND_RSMA2(pVnode);
return;
}
}
return VND_TSDB(pVnode);
*pTsdb = VND_TSDB(pVnode);
}
SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
@ -3631,6 +3710,10 @@ static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int6
}
TSDBKEY* p = taosArrayGet(pDelList, start);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
while (p->ts >= key && start > 0) {
start -= 1;
}
@ -3640,6 +3723,10 @@ static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int6
}
TSDBKEY* p = taosArrayGet(pDelList, start);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
while (p->ts <= key && start < num - 1) {
start += 1;
}
@ -3665,17 +3752,27 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
if (asc) {
if (*index >= num - 1) {
TSDBKEY* last = taosArrayGetLast(pDelList);
ASSERT(key >= last->ts);
if (last == NULL) {
return false;
}
ASSERT(key >= last->ts);
if (key > last->ts) {
return false;
} else if (key == last->ts) {
TSDBKEY* prev = taosArrayGet(pDelList, num - 2);
if (prev == NULL) {
return false;
}
return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer);
}
} else {
TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);
TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1);
if (pCurrent == NULL || pNext == NULL) {
return false;
}
if (key < pCurrent->ts) {
return false;
@ -3692,6 +3789,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
if ((*index) < num - 1) {
pCurrent = taosArrayGet(pDelList, *index);
pNext = taosArrayGet(pDelList, (*index) + 1);
if (pCurrent == NULL || pNext == NULL) {
return false;
}
// it is not a consecutive deletion range, ignore it
if (pCurrent->version == 0 && pNext->version > 0) {
@ -3710,6 +3810,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
} else {
if (*index <= 0) {
TSDBKEY* pFirst = taosArrayGet(pDelList, 0);
if (pFirst == NULL) {
return false;
}
if (key < pFirst->ts) {
return false;
@ -3721,6 +3824,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
} else {
TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);
TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1);
if (pCurrent == NULL || pPrev == NULL) {
return false;
}
if (key > pCurrent->ts) {
return false;
@ -3736,6 +3842,9 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
if ((*index) >= 1) {
pCurrent = taosArrayGet(pDelList, *index);
pPrev = taosArrayGet(pDelList, (*index) - 1);
if (pCurrent == NULL || pPrev == NULL) {
return false;
}
// it is not a consecutive deletion range, ignore it
if (pCurrent->version > 0 && pPrev->version == 0) {
@ -4229,6 +4338,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT
if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
i += 1;
}
@ -4238,6 +4351,9 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT
if (colId == pSchema->columns[j].colId) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
code = tRowGet(pTSRow, pSchema, j, &colVal);
if (code) {
@ -4252,6 +4368,9 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT
j += 1;
} else if (colId < pSchema->columns[j].colId) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
colDataSetNULL(pColInfoData, outputRowIndex);
i += 1;
@ -4263,6 +4382,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT
// set null value since current column does not exist in the "pSchema"
while (i < pSupInfo->numOfCols) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
if (pColInfoData == NULL) {
return TSDB_CODE_INVALID_PARA;
}
colDataSetNULL(pColInfoData, outputRowIndex);
i += 1;
}
@ -4311,6 +4434,10 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
while (i < numOfOutputCols) {
SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
if (pCol == NULL) {
return TSDB_CODE_INVALID_PARA;
}
colDataSetNULL(pCol, outputRowIndex);
i += 1;
}
@ -4494,19 +4621,16 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
STimeWindow window = pCond->twindows;
SVnodeCfg* pConf = &(((SVnode*)pVnode)->config);
int32_t code = 0;
int32_t lino = 0;
int32_t capacity = pConf->tsdbCfg.maxRows;
if (pResBlock != NULL) {
code = blockDataEnsureCapacity(pResBlock, capacity);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
TSDB_CHECK_CODE(code, lino, _err);
}
code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
TSDB_CHECK_CODE(code, lino, _err);
// check for query time window
STsdbReader* pReader = *ppReader;
@ -4530,9 +4654,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
// here we only need one more row, so the capacity is set to be ONE.
code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
TSDB_CHECK_CODE(code, lino, _err);
if (order == TSDB_ORDER_ASC) {
pCond->twindows.skey = window.ekey + 1;
@ -4544,9 +4666,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
pCond->order = order;
code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
TSDB_CHECK_CODE(code, lino, _err);
}
// NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here.
@ -4567,9 +4687,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
if (pReader->info.pSchema != NULL) {
code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
TSDB_CHECK_CODE(code, lino, _err);
}
pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash);
@ -4582,18 +4700,13 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc);
if (pReader->info.pSchema != NULL) {
code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
TSDB_CHECK_CODE(code, lino, _err);
}
STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader;
code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables, &pReader->status.pTableMap);
if (code != TSDB_CODE_SUCCESS) {
*ppReader = NULL;
goto _err;
}
TSDB_CHECK_CODE(code, lino, _err);
pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
if (pReader->status.pLDataIterArray == NULL) {

56
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -122,8 +122,10 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
size_t num = taosArrayGetSize(pBuf->pData);
for (int32_t i = 0; i < num; ++i) {
char** p = taosArrayGet(pBuf->pData, i);
if (p != NULL) {
taosMemoryFree(*p);
}
}
taosArrayDestroy(pBuf->pData);
}
@ -446,8 +448,8 @@ void cleanupInfoForNextFileset(SSHashObj* pTableMap) {
// brin records iterator
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) {
memset(&pIter->block, 0, sizeof(SBrinBlock));
memset(&pIter->record, 0, sizeof(SBrinRecord));
(void) memset(&pIter->block, 0, sizeof(SBrinBlock));
(void) memset(&pIter->record, 0, sizeof(SBrinRecord));
pIter->blockIndex = -1;
pIter->recordIndex = -1;
@ -465,6 +467,9 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
}
pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
if (pIter->pCurrentBlk == NULL) {
return TSDB_CODE_INVALID_PARA;
}
(void) tBrinBlockClear(&pIter->block);
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
@ -631,6 +636,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
for (int32_t k = 0; k < num; ++k) {
SFileDataBlockInfo* pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k);
if (pBlockInfo == NULL) {
return TSDB_CODE_INVALID_PARA;
}
sup.pDataBlockInfo[sup.numOfTables][k] =
(SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo};
cnt++;
@ -689,6 +698,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
int32_t index = sup.indexPerTable[pos]++;
SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index);
if (pBlockInfo == NULL) {
return TSDB_CODE_INVALID_PARA;
}
void* px = taosArrayPush(pBlockIter->blockList, pBlockInfo);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -1208,6 +1221,10 @@ bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlo
}
SSttKeyRange* pRange = taosArrayGet(pKeyRangeList, 0);
if (pRange == NULL) {
return false;
}
STimeWindow w = {.skey = pRange->skey.ts, .ekey = pRange->ekey.ts};
if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) {
return false;
@ -1216,6 +1233,9 @@ bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlo
for (int32_t i = 0; i < num - 1; ++i) {
SSttKeyRange* p1 = taosArrayGet(pKeyRangeList, i);
SSttKeyRange* p2 = taosArrayGet(pKeyRangeList, i + 1);
if (p1 == NULL || p2 == NULL) {
return false;
}
if (p1->ekey.ts >= p2->skey.ts) {
return false;
@ -1237,6 +1257,10 @@ static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const S
for (int32_t i = startIndex; i < num; i += 1) {
TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
if (p == NULL) {
return false;
}
if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {
if (p->version >= pRecord->minVer) {
return true;
@ -1245,6 +1269,10 @@ static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const S
if (p->version >= pRecord->minVer) {
if (i < num - 1) {
TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
if (pnext == NULL) {
return false;
}
if (pnext->ts >= pRecord->firstKey.key.ts) {
return true;
}
@ -1266,11 +1294,19 @@ static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockSca
for (int32_t i = startIndex; i < num; i += 1) {
TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
if (p == NULL) {
return false;
}
if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {
return true;
} else if (p->ts < pRecord->firstKey.key.ts) { // p->ts < pBlock->minKey.ts
if (i < num - 1) {
TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
if (pnext == NULL) {
return false;
}
if (pnext->ts >= pRecord->firstKey.key.ts) {
return true;
}
@ -1291,6 +1327,10 @@ bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecor
// ts is not overlap
TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
if (pFirst == NULL || pLast == NULL) {
return false;
}
if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {
return false;
}
@ -1302,6 +1342,10 @@ bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecor
int32_t index = pBlockScanInfo->fileDelIndex;
while (1) {
TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
if (p == NULL) {
return false;
}
if (p->ts > pRecord->firstKey.key.ts && index > 0) {
index -= 1;
} else { // find the first point that is smaller than the minKey.ts of dataBlock.
@ -1324,6 +1368,10 @@ bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const
// ts is not overlap
TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
if (pFirst == NULL || pLast == NULL) {
return false;
}
if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {
return false;
}
@ -1335,6 +1383,10 @@ bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const
int32_t index = pBlockScanInfo->fileDelIndex;
while (1) {
TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
if (p == NULL) {
return false;
}
if (p->ts > pRecord->firstKey.key.ts && index > 0) {
index -= 1;
} else { // find the first point that is smaller than the minKey.ts of dataBlock.

7
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -757,10 +757,9 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m
TSDB_CHECK_CODE(code, lino, _exit);
// decode
int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk);
if (n < 0) {
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
int32_t n;
code = tGetMapData(pReader->aBuf[0], mDataBlk, &n);
if (code) goto _exit;
ASSERT(n == size);
_exit:

181
source/dnode/vnode/src/tsdb/tsdbSnapInfo.c
View File

@ -42,7 +42,6 @@ static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) {
STsdbFSetPartition* tsdbFSetPartitionCreate() {
STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition));
if (pSP == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) {
@ -81,7 +80,7 @@ static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) {
static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) {
STsdbFSetPartition* p = tsdbFSetPartitionCreate();
if (p == NULL) {
goto _err;
return terrno;
}
p->fid = fset->fid;
@ -134,17 +133,12 @@ static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition**
}
ppSP[0] = p;
return 0;
_err:
tsdbFSetPartitionClear(&p);
return -1;
}
// fset partition list
STsdbFSetPartList* tsdbFSetPartListCreate() {
STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList));
if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
TARRAY2_INIT(pList);
@ -160,9 +154,11 @@ void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) {
}
int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) {
int32_t code = 0;
TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray));
if (pDiff == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _err;
}
TARRAY2_INIT(pDiff);
@ -171,7 +167,7 @@ int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray
TARRAY2_FOREACH(pList, part) {
STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange));
if (r == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _err;
}
int64_t maxVerValid = -1;
@ -202,7 +198,7 @@ _err:
if (pDiff) {
tsdbTFileSetRangeArrayDestroy(&pDiff);
}
return -1;
return code;
}
// serialization
@ -235,96 +231,103 @@ int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) {
return datLen;
}
int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
static int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList, int32_t* encodeSize) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
int8_t reserved8 = 0;
int16_t reserved16 = 0;
int64_t reserved64 = 0;
int8_t msgVer = TSDB_SNAP_MSG_VER;
int32_t len = TARRAY2_SIZE(pList);
int32_t code = 0;
if (tStartEncode(&encoder) < 0) goto _err;
if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
if (tEncodeI32(&encoder, len) < 0) goto _err;
tEncoderInit(&encoder, buf, bufLen);
if ((code = tStartEncode(&encoder))) goto _exit;
if ((code = tEncodeI8(&encoder, msgVer))) goto _exit;
if ((code = tEncodeI32(&encoder, len))) goto _exit;
for (int32_t u = 0; u < len; u++) {
STsdbFSetPartition* p = TARRAY2_GET(pList, u);
if (tEncodeI64(&encoder, p->fid) < 0) goto _err;
if (tEncodeI8(&encoder, p->stat) < 0) goto _err;
if (tEncodeI8(&encoder, reserved8) < 0) goto _err;
if (tEncodeI16(&encoder, reserved16) < 0) goto _err;
if ((code = tEncodeI64(&encoder, p->fid))) goto _exit;
if ((code = tEncodeI8(&encoder, p->stat))) goto _exit;
if ((code = tEncodeI8(&encoder, reserved8))) goto _exit;
if ((code = tEncodeI16(&encoder, reserved16))) goto _exit;
int32_t typMax = TSDB_FSET_RANGE_TYP_MAX;
if (tEncodeI32(&encoder, typMax) < 0) goto _err;
if ((code = tEncodeI32(&encoder, typMax))) goto _exit;
for (int32_t i = 0; i < typMax; i++) {
SVerRangeList* iList = &p->verRanges[i];
int32_t iLen = TARRAY2_SIZE(iList);
if (tEncodeI32(&encoder, iLen) < 0) goto _err;
if ((code = tEncodeI32(&encoder, iLen))) goto _exit;
for (int32_t j = 0; j < iLen; j++) {
SVersionRange r = TARRAY2_GET(iList, j);
if (tEncodeI64(&encoder, r.minVer) < 0) goto _err;
if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err;
if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
if ((code = tEncodeI64(&encoder, r.minVer))) goto _exit;
if ((code = tEncodeI64(&encoder, r.maxVer))) goto _exit;
if ((code = tEncodeI64(&encoder, reserved64))) goto _exit;
}
}
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
_err:
if (encodeSize) {
encodeSize[0] = encoder.pos;
}
_exit:
tEncoderClear(&encoder);
return -1;
return code;
}
int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
int8_t reserved8 = 0;
int16_t reserved16 = 0;
int64_t reserved64 = 0;
int32_t code = 0;
STsdbFSetPartition* p = NULL;
tDecoderInit(&decoder, buf, bufLen);
int8_t msgVer = 0;
int32_t len = 0;
if (tStartDecode(&decoder) < 0) goto _err;
if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
if (tDecodeI32(&decoder, &len) < 0) goto _err;
if ((code = tStartDecode(&decoder))) goto _err;
if ((code = tDecodeI8(&decoder, &msgVer))) goto _err;
if (msgVer != TSDB_SNAP_MSG_VER) {
code = TSDB_CODE_INVALID_MSG;
goto _err;
}
if ((code = tDecodeI32(&decoder, &len))) goto _err;
for (int32_t u = 0; u < len; u++) {
p = tsdbFSetPartitionCreate();
if (p == NULL) goto _err;
if (tDecodeI64(&decoder, &p->fid) < 0) goto _err;
if (tDecodeI8(&decoder, &p->stat) < 0) goto _err;
if (tDecodeI8(&decoder, &reserved8) < 0) goto _err;
if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
if (p == NULL) {
code = terrno;
goto _err;
}
if ((code = tDecodeI64(&decoder, &p->fid))) goto _err;
if ((code = tDecodeI8(&decoder, &p->stat))) goto _err;
if ((code = tDecodeI8(&decoder, &reserved8))) goto _err;
if ((code = tDecodeI16(&decoder, &reserved16))) goto _err;
int32_t typMax = 0;
if (tDecodeI32(&decoder, &typMax) < 0) goto _err;
if ((code = tDecodeI32(&decoder, &typMax))) goto _err;
for (int32_t i = 0; i < typMax; i++) {
SVerRangeList* iList = &p->verRanges[i];
int32_t iLen = 0;
if (tDecodeI32(&decoder, &iLen) < 0) goto _err;
if ((code = tDecodeI32(&decoder, &iLen))) goto _err;
for (int32_t j = 0; j < iLen; j++) {
SVersionRange r = {0};
if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
if (TARRAY2_APPEND(iList, r)) goto _err;
if ((code = tDecodeI64(&decoder, &r.minVer))) goto _err;
if ((code = tDecodeI64(&decoder, &r.maxVer))) goto _err;
if ((code = tDecodeI64(&decoder, &reserved64))) goto _err;
if ((code = TARRAY2_APPEND(iList, r))) goto _err;
}
}
if (TARRAY2_APPEND(pList, p)) goto _err;
if ((code = TARRAY2_APPEND(pList, p))) goto _err;
p = NULL;
}
@ -337,7 +340,7 @@ _err:
tsdbFSetPartitionClear(&p);
}
tDecoderClear(&decoder);
return -1;
return code;
}
// fs state
@ -352,8 +355,9 @@ static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) {
STFileSet* fset;
TARRAY2_FOREACH(fs->fSetArr, fset) {
STsdbFSetPartition* pItem = NULL;
if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) {
code = -1;
code = tsdbTFileSetToFSetPartition(fset, &pItem);
if (code) {
terrno = code;
break;
}
ASSERT(pItem != NULL);
@ -404,7 +408,9 @@ static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo)
for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS);
if (pInfo->pLists[j] == NULL) return -1;
if (pInfo->pLists[j] == NULL) {
return terrno;
}
}
return 0;
}
@ -432,9 +438,10 @@ static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* bu
SSyncTLV* pSubHead = (void*)((char*)buf + offset);
int32_t valOffset = offset + sizeof(*pSubHead);
ASSERT(pSubHead->val == (char*)buf + valOffset);
if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) {
int32_t code = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j], &tlen);
if (code) {
tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr());
return -1;
return code;
}
pSubHead->typ = pInfo->subTyps[j];
pSubHead->len = tlen;
@ -460,17 +467,18 @@ static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) {
}
int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
int32_t code = 0;
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
int64_t reserved64 = 0;
int8_t msgVer = TSDB_SNAP_MSG_VER;
if (tStartEncode(&encoder) < 0) goto _err;
if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
tEncoderInit(&encoder, buf, bufLen);
if ((code = tStartEncode(&encoder))) goto _err;
if ((code = tEncodeI8(&encoder, msgVer))) goto _err;
int16_t format = pOpts->format;
if (tEncodeI16(&encoder, format) < 0) goto _err;
if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
if ((code = tEncodeI16(&encoder, format))) goto _err;
if ((code = tEncodeI64(&encoder, reserved64))) goto _err;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -479,23 +487,24 @@ int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
_err:
tEncoderClear(&encoder);
return -1;
return code;
}
int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
int32_t code;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
int64_t reserved64 = 0;
int8_t msgVer = 0;
if (tStartDecode(&decoder) < 0) goto _err;
if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
tDecoderInit(&decoder, buf, bufLen);
if ((code = tStartDecode(&decoder))) goto _err;
if ((code = tDecodeI8(&decoder, &msgVer))) goto _err;
if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
int16_t format = 0;
if (tDecodeI16(&decoder, &format) < 0) goto _err;
if ((code = tDecodeI16(&decoder, &format))) goto _err;
pOpts->format = format;
if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
if ((code = tDecodeI64(&decoder, &reserved64))) goto _err;
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -503,7 +512,7 @@ int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts)
_err:
tDecoderClear(&decoder);
return -1;
return code;
}
static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) {
@ -518,7 +527,7 @@ static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufL
int32_t offset = 0;
int32_t tlen = 0;
if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) {
return -1;
return tlen;
}
pSubHead->typ = SNAP_DATA_RAW;
pSubHead->len = tlen;
@ -528,8 +537,10 @@ static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufL
// snap info
static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) {
if (!pSnap->data) return 0;
int32_t code = -1;
if (!pSnap->data) {
return 0;
}
int32_t code = 0;
SSyncTLV* pHead = (void*)pSnap->data;
int32_t offset = 0;
@ -546,30 +557,30 @@ static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, ST
case SNAP_DATA_RSMA2: {
} break;
case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
code = tDeserializeTsdbRepOpts(buf, bufLen, pInfo);
if (code < 0) {
tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out;
return code;
}
} break;
default:
code = TSDB_CODE_INVALID_MSG;
tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ);
goto _out;
return code;
}
}
code = 0;
_out:
return code;
}
int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY);
STsdbPartitionInfo partitionInfo = {0};
int code = -1;
int code = 0;
STsdbPartitionInfo* pInfo = &partitionInfo;
if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) {
code = tsdbPartitionInfoInit(pVnode, pInfo);
if (code) {
goto _out;
}
@ -577,7 +588,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW};
if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
STsdbRepOpts leaderOpts = {0};
if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) {
if ((code = tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts)) < 0) {
tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr());
goto _out;
}
@ -589,7 +600,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
int32_t bufLen = headLen;
bufLen += tsdbPartitionInfoEstSize(pInfo);
bufLen += tsdbRepOptsEstSize(&opts);
if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) {
if ((code = syncSnapInfoDataRealloc(pSnap, bufLen)) != 0) {
tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen);
goto _out;
}
@ -599,7 +610,8 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
int32_t offset = headLen;
int32_t tlen = 0;
if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) {
if ((tlen = tsdbPartitionInfoSerialize(pInfo, (uint8_t*)(buf + offset), bufLen - offset)) < 0) {
code = tlen;
tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr());
goto _out;
}
@ -607,6 +619,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
ASSERT(offset <= bufLen);
if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) {
code = tlen;
tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out;
}
@ -620,7 +633,7 @@ int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ),
pHead->len);
code = 0;
_out:
tsdbPartitionInfoClear(pInfo);
return code;

104
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -152,15 +152,19 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
}
#endif
int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
int32_t tGetMapData(uint8_t *p, SMapData *pMapData, int32_t *decodedSize) {
int32_t n = 0;
int32_t code;
int32_t offset;
tMapDataReset(pMapData);
n += tGetI32v(p + n, &pMapData->nItem);
if (pMapData->nItem) {
if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;
code = tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem);
if (code) {
return code;
}
int32_t lOffset = 0;
for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
@ -170,12 +174,19 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
}
n += tGetI32v(p + n, &pMapData->nData);
if (tRealloc(&pMapData->pData, pMapData->nData)) return -1;
code = tRealloc(&pMapData->pData, pMapData->nData);
if (code) {
return code;
}
memcpy(pMapData->pData, p + n, pMapData->nData);
n += pMapData->nData;
}
return n;
if (decodedSize) {
*decodedSize = n;
}
return 0;
}
#ifdef BUILD_NO_CALL
@ -680,20 +691,17 @@ int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2) {
// STSDBRowIter ======================================================
int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) {
int32_t code = 0;
pIter->pRow = pRow;
if (pRow->type == TSDBROW_ROW_FMT) {
code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter);
if (code) goto _exit;
int32_t code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter);
if (code) return code;
} else if (pRow->type == TSDBROW_COL_FMT) {
pIter->iColData = 0;
} else {
ASSERT(0);
}
_exit:
return code;
return 0;
}
void tsdbRowClose(STSDBRowIter *pIter) {
@ -748,7 +756,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
*pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = key.ts}));
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
return code;
}
@ -760,7 +768,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
--iCol;
continue;
} else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) {
return terrno;
}
continue;
}
@ -771,7 +781,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
pColVal->value.pData = NULL;
code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
if (code) {
return TSDB_CODE_OUT_OF_MEMORY;
return code;
}
if (pColVal->value.nData) {
@ -780,14 +790,15 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema)
}
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
return terrno;
}
}
for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol];
taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
if (taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)) == NULL) {
return terrno;
}
}
pMerger->version = key.version;
@ -858,7 +869,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema) {
pMerger->pTSchema = pSchema;
pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal));
if (pMerger->pArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
} else {
return TSDB_CODE_SUCCESS;
}
@ -962,8 +973,7 @@ _exit:
*/
// delete skyline ======================================================
static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) {
int32_t code = 0;
static void tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pSkyline) {
int32_t i1 = 0;
int32_t n1 = taosArrayGetSize(pSkyline1);
int32_t i2 = 0;
@ -1017,7 +1027,6 @@ static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pS
}
pSkyline->size = TARRAY_ELEM_IDX(pSkyline, pItem);
return code;
}
int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, SArray *pSkyline) {
@ -1029,18 +1038,26 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx,
if (sidx == eidx) {
TSDBKEY *pItem1 = taosArrayGet(aSkyline, sidx * 2);
TSDBKEY *pItem2 = taosArrayGet(aSkyline, sidx * 2 + 1);
taosArrayPush(pSkyline, &pItem1);
taosArrayPush(pSkyline, &pItem2);
if (taosArrayPush(pSkyline, &pItem1) == NULL) {
return terrno;
}
if (taosArrayPush(pSkyline, &pItem2) == NULL) {
return terrno;
}
} else {
SArray *pSkyline1 = NULL;
SArray *pSkyline2 = NULL;
midx = (sidx + eidx) / 2;
pSkyline1 = taosArrayInit((midx - sidx + 1) * 2, POINTER_BYTES);
if (pSkyline1 == NULL) {
return terrno;
}
pSkyline2 = taosArrayInit((eidx - midx) * 2, POINTER_BYTES);
if (pSkyline1 == NULL || pSkyline1 == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _clear;
if (pSkyline2 == NULL) {
taosArrayDestroy(pSkyline1);
return terrno;
}
code = tsdbBuildDeleteSkylineImpl(aSkyline, sidx, midx, pSkyline1);
@ -1049,7 +1066,7 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx,
code = tsdbBuildDeleteSkylineImpl(aSkyline, midx + 1, eidx, pSkyline2);
if (code) goto _clear;
code = tsdbMergeSkyline(pSkyline1, pSkyline2, pSkyline);
tsdbMergeSkyline(pSkyline1, pSkyline2, pSkyline);
_clear:
taosArrayDestroy(pSkyline1);
@ -1064,13 +1081,28 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
int32_t code = 0;
int32_t dataNum = eidx - sidx + 1;
SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY));
if (aTmpSkyline == NULL) {
return terrno;
}
SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES);
if (pSkyline == NULL) {
taosArrayDestroy(aTmpSkyline);
return terrno;
}
taosArrayClear(aSkyline);
for (int32_t i = sidx; i <= eidx; ++i) {
pDelData = (SDelData *)taosArrayGet(aDelData, i);
taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version});
taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0});
if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}) == NULL) {
code = terrno;
goto _clear;
}
if (taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->eKey, .version = 0}) == NULL) {
code = terrno;
goto _clear;
}
}
code = tsdbBuildDeleteSkylineImpl(aTmpSkyline, sidx, eidx, pSkyline);
@ -1079,7 +1111,10 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
int32_t skylineNum = taosArrayGetSize(pSkyline);
for (int32_t i = 0; i < skylineNum; ++i) {
TSDBKEY *p = taosArrayGetP(pSkyline, i);
taosArrayPush(aSkyline, p);
if (taosArrayPush(aSkyline, p) == NULL) {
code = terrno;
goto _clear;
}
}
_clear:
@ -1167,7 +1202,7 @@ static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData)
} else if (pBlockData->nColData < nColData) {
SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData);
if (aColData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -1255,7 +1290,7 @@ int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, i
SColData *newColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SColData) * (pBlockData->nColData + 1));
if (newColData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pBlockData->aColData = newColData;
@ -1423,6 +1458,7 @@ int32_t tBlockDataCompress(SBlockData *bData, void *pCompr, SBuffer *buffers, SB
SColCompressInfo *pInfo = pCompr;
code = tsdbGetColCmprAlgFromSet(pInfo->pColCmpr, 1, &pInfo->defaultCmprAlg);
TAOS_UNUSED(code);
SDiskDataHdr hdr = {
.delimiter = TSDB_FILE_DLMT,
@ -1519,8 +1555,8 @@ int32_t tBlockDataDecompress(SBufferReader *br, SBlockData *blockData, SBuffer *
SBlockCol blockCol;
code = tGetBlockCol(&br2, &blockCol, hdr.fmtVer, hdr.cmprAlg);
if (blockCol.alg == 0) blockCol.alg = hdr.cmprAlg;
TSDB_CHECK_CODE(code, lino, _exit);
if (blockCol.alg == 0) blockCol.alg = hdr.cmprAlg;
code = tBlockDataDecompressColData(&hdr, &blockCol, br, blockData, assist);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -1812,7 +1848,9 @@ int32_t tsdbGetColCmprAlgFromSet(SHashObj *set, int16_t colId, uint32_t *alg) {
if (set == NULL) return -1;
uint32_t *ret = taosHashGet(set, &colId, sizeof(colId));
if (ret == NULL) return -1;
if (ret == NULL) {
return TSDB_CODE_NOT_FOUND;
}
*alg = *ret;
return 0;

6
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -140,8 +140,9 @@ void *vnodeBufPoolMallocAligned(SVBufPool *pPool, int size) {
// allocate a new node
pNode = taosMemoryMalloc(sizeof(*pNode) + size);
if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
if (pPool->lock) taosThreadSpinUnlock(pPool->lock);
if (pPool->lock) {
taosThreadSpinUnlock(pPool->lock);
}
return NULL;
}
@ -173,7 +174,6 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) {
// allocate a new node
pNode = taosMemoryMalloc(sizeof(*pNode) + size);
if (pNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
if (pPool->lock) taosThreadSpinUnlock(pPool->lock);
return NULL;
}

254
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -88,33 +88,33 @@ const ESyncRole vnodeStrToRole(char *str) {
int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
const SVnodeCfg *pCfg = (SVnodeCfg *)pObj;
if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1;
if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId));
TAOS_CHECK_RETURN(tjsonAddStringToObject(pJson, "dbname", pCfg->dbname));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szPage", pCfg->szPage));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLast", pCfg->cacheLast));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "cacheLastSize", pCfg->cacheLastSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isTsma", pCfg->isTsma));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "isRsma", pCfg->isRsma));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "slLevel", pCfg->tsdbCfg.slLevel));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3ChunkSize", pCfg->s3ChunkSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3KeepLocal", pCfg->s3KeepLocal));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "s3Compact", pCfg->s3Compact));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize));
if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@ -124,61 +124,67 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
}
}
SJson *pNodeRetentions = tjsonCreateArray();
if (pNodeRetentions == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tjsonAddItemToObject(pJson, "retentions", pNodeRetentions);
for (int32_t i = 0; i < nRetention; ++i) {
SJson *pNodeRetention = tjsonCreateObject();
const SRetention *pRetention = pCfg->tsdbCfg.retentions + i;
tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq);
tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit);
tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep);
tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit);
tjsonAddItemToArray(pNodeRetentions, pNodeRetention);
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freq", pRetention->freq));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "freqUnit", pRetention->freqUnit));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keep", pRetention->keep));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pNodeRetention, "keepUnit", pRetention->keepUnit));
TAOS_CHECK_RETURN(tjsonAddItemToArray(pNodeRetentions, pNodeRetention));
}
}
if (tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashChange", pCfg->hashChange));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries));
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
if (nodeInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
TAOS_CHECK_RETURN(tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo));
vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d changeVersion:%d", pCfg->vgId,
pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex, pCfg->syncCfg.changeVersion);
for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) {
SJson *info = tjsonCreateObject();
if (info == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1;
if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1;
if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1;
if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1;
if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1;
if (tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1;
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort));
TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId));
TAOS_CHECK_RETURN(tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)));
TAOS_CHECK_RETURN(tjsonAddItemToArray(nodeInfo, info));
vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId);
}
@ -191,50 +197,50 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
int32_t code;
tjsonGetNumberValue(pJson, "vgId", pCfg->vgId, code);
if (code < 0) return -1;
if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1;
if (code) return code;
if ((code = tjsonGetStringValue(pJson, "dbname", pCfg->dbname))) return code;
tjsonGetNumberValue(pJson, "dbId", pCfg->dbId, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "szPage", pCfg->szPage, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "szCache", pCfg->szCache, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "cacheLast", pCfg->cacheLast, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "cacheLastSize", pCfg->cacheLastSize, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isTsma", pCfg->isTsma, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "isRsma", pCfg->isRsma, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "slLevel", pCfg->tsdbCfg.slLevel, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset, code);
if (code < 0) return -1;
if (code) return code;
SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
if (nRetention > TSDB_RETENTION_MAX) {
@ -244,52 +250,54 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i);
ASSERT(pNodeRetention != NULL);
tjsonGetNumberValue(pNodeRetention, "freq", (pCfg->tsdbCfg.retentions)[i].freq, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "freqUnit", (pCfg->tsdbCfg.retentions)[i].freqUnit, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep, code);
if (code) return code;
tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit, code);
if (code) return code;
}
tjsonGetNumberValue(pJson, "tsdb.encryptAlgorithm", pCfg->tsdbCfg.encryptAlgorithm, code);
if (code < 0) return -1;
if (code) return code;
#if defined(TD_ENTERPRISE)
if (pCfg->tsdbCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
} else {
strncpy(pCfg->tsdbCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.clearFiles", pCfg->walCfg.clearFiles, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "wal.encryptAlgorithm", pCfg->walCfg.encryptAlgorithm, code);
if (code < 0) return -1;
if (code) return code;
#if defined(TD_ENTERPRISE)
if (pCfg->walCfg.encryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
return -1;
return terrno = TSDB_CODE_DNODE_INVALID_ENCRYPTKEY;
} else {
strncpy(pCfg->walCfg.encryptKey, tsEncryptKey, ENCRYPT_KEY_LEN);
}
}
#endif
tjsonGetNumberValue(pJson, "tdbEncryptAlgorithm", pCfg->tdbEncryptAlgorithm, code);
if (code < 0) return -1;
if (code) return code;
#if defined(TD_ENTERPRISE)
if (pCfg->tdbEncryptAlgorithm == DND_CA_SM4) {
if (tsEncryptKey[0] == 0) {
@ -303,35 +311,35 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashChange", pCfg->hashChange, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code);
if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code);
if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "syncCfg.changeVersion", pCfg->syncCfg.changeVersion, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ctables", pCfg->vndStats.numOfCTables, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntables", pCfg->vndStats.numOfNTables, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code);
if (code < 0) return -1;
if (code) return code;
SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(nodeInfo);
@ -344,15 +352,15 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1;
tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn);
tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code);
if (code < 0) return -1;
if (code) return code;
char role[10] = {0};
code = tjsonGetStringValue(info, "isReplica", role);
if (code < 0) return -1;
if (code) return code;
if (strlen(role) != 0) {
pNode->nodeRole = vnodeStrToRole(role);
} else {

182
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -125,20 +125,16 @@ int vnodeBegin(SVnode *pVnode) {
TSDB_CHECK_CODE(code, lino, _exit);
// begin meta
if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) {
code = terrno;
code = metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL);
TSDB_CHECK_CODE(code, lino, _exit);
}
// begin tsdb
if (tsdbBegin(pVnode->pTsdb) < 0) {
code = terrno;
code = tsdbBegin(pVnode->pTsdb);
TSDB_CHECK_CODE(code, lino, _exit);
}
// begin sma
if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) {
code = terrno;
if (VND_IS_RSMA(pVnode)) {
code = smaBegin(pVnode->pSma);
TSDB_CHECK_CODE(code, lino, _exit);
}
@ -171,54 +167,41 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
}
int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile;
char *data;
TdFilePtr pFile = NULL;
char *data = NULL;
snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
// encode info
data = NULL;
if (vnodeEncodeInfo(pInfo, &data) < 0) {
vError("failed to encode json info.");
return -1;
}
code = vnodeEncodeInfo(pInfo, &data);
TSDB_CHECK_CODE(code, lino, _exit);
// save info to a vnode_tmp.json
pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH);
if (pFile == NULL) {
vError("failed to open info file:%s for write:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
if (taosWriteFile(pFile, data, strlen(data)) < 0) {
vError("failed to write info file:%s error:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
if (taosFsyncFile(pFile) < 0) {
vError("failed to fsync info file:%s error:%s", fname, terrstr());
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
taosCloseFile(&pFile);
// free info binary
taosMemoryFree(data);
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
} else {
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d changeVersion:%d", pInfo->config.vgId, fname,
pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex, pInfo->config.syncCfg.changeVersion);
return 0;
_err:
}
taosCloseFile(&pFile);
taosMemoryFree(data);
return -1;
return code;
}
int vnodeCommitInfo(const char *dir) {
@ -229,8 +212,7 @@ int vnodeCommitInfo(const char *dir) {
snprintf(tfname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
if (taosRenameFile(tfname, fname) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
return terrno = TAOS_SYSTEM_ERROR(errno);
}
vInfo("vnode info is committed, dir:%s", dir);
@ -238,6 +220,8 @@ int vnodeCommitInfo(const char *dir) {
}
int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
char fname[TSDB_FILENAME_LEN];
TdFilePtr pFile = NULL;
char *pData = NULL;
@ -248,44 +232,35 @@ int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
// read info
pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
if (taosReadFile(pFile, pData, size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
pData[size] = '\0';
taosCloseFile(&pFile);
// decode info
if (vnodeDecodeInfo(pData, pInfo) < 0) {
taosMemoryFree(pData);
return -1;
code = vnodeDecodeInfo(pData, pInfo);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", pInfo->config.vgId, __func__, __FILE__, lino, tstrerror(code));
}
taosMemoryFree(pData);
return 0;
_err:
taosCloseFile(&pFile);
taosMemoryFree(pData);
return -1;
return code;
}
static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
@ -297,7 +272,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
// wait last commit task
vnodeAWait(&pVnode->commitTask);
if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit;
code = syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg);
TSDB_CHECK_CODE(code, lino, _exit);
pVnode->state.commitTerm = pVnode->state.applyTerm;
@ -312,17 +288,16 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN);
vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode));
if (vnodeSaveInfo(dir, &pInfo->info) < 0) {
code = terrno;
code = vnodeSaveInfo(dir, &pInfo->info);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdbPreCommit(pVnode->pTsdb);
metaPrepareAsyncCommit(pVnode->pMeta);
code = metaPrepareAsyncCommit(pVnode->pMeta);
TSDB_CHECK_CODE(code, lino, _exit);
code = smaPrepareAsyncCommit(pVnode->pSma);
if (code) goto _exit;
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pVnode->mutex);
ASSERT(pVnode->onCommit == NULL);
@ -459,10 +434,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
}
// commit info
if (vnodeCommitInfo(dir) < 0) {
code = terrno;
code = vnodeCommitInfo(dir);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbCommitCommit(pVnode->pTsdb);
TSDB_CHECK_CODE(code, lino, _exit);
@ -472,10 +445,8 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) {
TSDB_CHECK_CODE(code, lino, _exit);
}
if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) {
code = terrno;
code = metaFinishCommit(pVnode->pMeta, pInfo->txn);
TSDB_CHECK_CODE(code, lino, _exit);
}
pVnode->state.committed = pInfo->info.state.committed;
@ -492,7 +463,7 @@ _exit:
} else {
vInfo("vgId:%d, commit end", TD_VID(pVnode));
}
return 0;
return code;
}
bool vnodeShouldRollback(SVnode *pVnode) {
@ -514,15 +485,15 @@ void vnodeRollback(SVnode *pVnode) {
offset = strlen(tFName);
snprintf(tFName + offset, TSDB_FILENAME_LEN - offset - 1, "%s%s", TD_DIRSEP, VND_INFO_FNAME_TMP);
(void)taosRemoveFile(tFName);
TAOS_UNUSED(taosRemoveFile(tFName));
}
static int vnodeEncodeState(const void *pObj, SJson *pJson) {
const SVState *pState = (SVState *)pObj;
if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1;
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit version", pState->committed));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID));
TAOS_CHECK_RETURN(tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm));
return 0;
}
@ -532,70 +503,67 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
int32_t code;
tjsonGetNumberValue(pJson, "commit version", pState->committed, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
if (code < 0) return -1;
if (code) return code;
tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
if (code < 0) return -1;
if (code) return code;
return 0;
}
static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
SJson *pJson;
char *pData;
*ppData = NULL;
int32_t code = 0;
int32_t lino;
SJson *pJson = NULL;
char *pData = NULL;
pJson = tjsonCreateObject();
if (pJson == NULL) {
return -1;
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
if (tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config) < 0) {
goto _err;
}
code = tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config);
TSDB_CHECK_CODE(code, lino, _exit);
if (tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state) < 0) {
goto _err;
}
code = tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state);
TSDB_CHECK_CODE(code, lino, _exit);
pData = tjsonToString(pJson);
if (pData == NULL) {
goto _err;
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
tjsonDelete(pJson);
*ppData = pData;
return 0;
_err:
_exit:
if (code) {
tjsonDelete(pJson);
return -1;
*ppData = NULL;
} else {
*ppData = pData;
}
return code;
}
int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
int32_t code = 0;
int32_t lino;
SJson *pJson = NULL;
pJson = tjsonParse(pData);
if (pJson == NULL) {
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
return -1;
}
if (tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config) < 0) {
goto _err;
}
code = tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config);
TSDB_CHECK_CODE(code, lino, _exit);
if (tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state) < 0) {
goto _err;
}
code = tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
tjsonDelete(pJson);
return 0;
_err:
tjsonDelete(pJson);
return -1;
return code;
}

182
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -47,7 +47,7 @@ int32_t fillTableColCmpr(SMetaReader *reader, SSchemaExt *pExt, int32_t numOfCol
return 0;
}
int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
int32_t vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
STableInfoReq infoReq = {0};
STableMetaRsp metaRsp = {0};
SMetaReader mer1 = {0};
@ -62,15 +62,15 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req
if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
code = terrno;
goto _exit4;
}
metaRsp.dbId = pVnode->config.dbId;
strcpy(metaRsp.tbName, infoReq.tbName);
memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
(void)strcpy(metaRsp.tbName, infoReq.tbName);
(void)memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
(void)sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName);
if (code) {
goto _exit4;
@ -89,7 +89,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.tuid = mer1.me.uid;
if (mer1.me.type == TSDB_SUPER_TABLE) {
strcpy(metaRsp.stbName, mer1.me.name);
(void)strcpy(metaRsp.stbName, mer1.me.name);
schema = mer1.me.stbEntry.schemaRow;
schemaTag = mer1.me.stbEntry.schemaTag;
metaRsp.suid = mer1.me.uid;
@ -97,7 +97,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2;
strcpy(metaRsp.stbName, mer2.me.name);
(void)strcpy(metaRsp.stbName, mer2.me.name);
metaRsp.suid = mer2.me.uid;
schema = mer2.me.stbEntry.schemaRow;
schemaTag = mer2.me.stbEntry.schemaTag;
@ -114,10 +114,13 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaRsp.tversion = schemaTag.version;
metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags));
metaRsp.pSchemaExt = (SSchemaExt *)taosMemoryCalloc(metaRsp.numOfColumns, sizeof(SSchemaExt));
memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (NULL == metaRsp.pSchemas || NULL == metaRsp.pSchemaExt) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) {
memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
(void)memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
}
if (metaRsp.pSchemaExt) {
SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
@ -134,7 +137,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// encode and send response
rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG;
code = terrno;
goto _exit;
}
@ -148,7 +151,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
rspLen = tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
if (rspLen < 0) {
code = terrno;
goto _exit;
}
_exit:
taosMemoryFree(metaRsp.pSchemas);
@ -174,10 +182,10 @@ _exit4:
*pMsg = rpcMsg;
}
return TSDB_CODE_SUCCESS;
return code;
}
int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
STableCfgReq cfgReq = {0};
STableCfgRsp cfgRsp = {0};
SMetaReader mer1 = {0};
@ -192,14 +200,14 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// decode req
if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) {
code = TSDB_CODE_INVALID_MSG;
code = terrno;
goto _exit;
}
strcpy(cfgRsp.tbName, cfgReq.tbName);
memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName));
(void)strcpy(cfgRsp.tbName, cfgReq.tbName);
(void)memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName));
sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName);
(void)sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName);
code = vnodeValidateTableHash(pVnode, tableFName);
if (code) {
goto _exit;
@ -222,24 +230,36 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK);
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
strcpy(cfgRsp.stbName, mer2.me.name);
(void)strcpy(cfgRsp.stbName, mer2.me.name);
schema = mer2.me.stbEntry.schemaRow;
schemaTag = mer2.me.stbEntry.schemaTag;
cfgRsp.ttl = mer1.me.ctbEntry.ttlDays;
cfgRsp.commentLen = mer1.me.ctbEntry.commentLen;
if (mer1.me.ctbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
STag *pTag = (STag *)mer1.me.ctbEntry.pTags;
cfgRsp.tagsLen = pTag->len;
cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen);
memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen);
if (NULL == cfgRsp.pTags) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen);
} else if (mer1.me.type == TSDB_NORMAL_TABLE) {
schema = mer1.me.ntbEntry.schemaRow;
cfgRsp.ttl = mer1.me.ntbEntry.ttlDays;
cfgRsp.commentLen = mer1.me.ntbEntry.commentLen;
if (mer1.me.ntbEntry.commentLen > 0) {
cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment);
if (NULL == cfgRsp.pComment) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
} else {
ASSERT(0);
@ -250,9 +270,13 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags));
cfgRsp.pSchemaExt = (SSchemaExt *)taosMemoryMalloc(cfgRsp.numOfColumns * sizeof(SSchemaExt));
memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (NULL == cfgRsp.pSchemas || NULL == cfgRsp.pSchemaExt) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
(void)memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols);
if (schemaTag.nCols) {
memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
(void)memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
}
// if (useCompress(cfgRsp.tableType)) {
@ -271,7 +295,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
// encode and send response
rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp);
if (rspLen < 0) {
code = TSDB_CODE_INVALID_MSG;
code = terrno;
goto _exit;
}
@ -285,7 +309,12 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp);
rspLen = tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp);
if (rspLen < 0) {
code = terrno;
goto _exit;
}
_exit:
rpcMsg.info = pMsg->info;
@ -307,7 +336,7 @@ _exit:
tFreeSTableCfgRsp(&cfgRsp);
metaReaderClear(&mer2);
metaReaderClear(&mer1);
return TSDB_CODE_SUCCESS;
return code;
}
static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) {
@ -331,7 +360,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
void *pRsp = NULL;
if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
qError("tDeserializeSBatchReq failed");
goto _exit;
}
@ -352,6 +381,10 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
for (int32_t i = 0; i < msgNum; ++i) {
req = taosArrayGet(batchReq.pMsgs, i);
if (req == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
goto _exit;
}
reqMsg.msgType = req->msgType;
reqMsg.pCont = req->msg;
@ -359,13 +392,16 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
switch (req->msgType) {
case TDMT_VND_TABLE_META:
vnodeGetTableMeta(pVnode, &reqMsg, false);
// error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetTableMeta(pVnode, &reqMsg, false);
break;
case TDMT_VND_TABLE_CFG:
vnodeGetTableCfg(pVnode, &reqMsg, false);
// error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetTableCfg(pVnode, &reqMsg, false);
break;
case TDMT_VND_GET_STREAM_PROGRESS:
vnodeGetStreamProgress(pVnode, &reqMsg, false);
// error code has been set into reqMsg, no need to handle it here.
(void)vnodeGetStreamProgress(pVnode, &reqMsg, false);
break;
default:
qError("invalid req msgType %d", req->msgType);
@ -381,24 +417,28 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
rsp.rspCode = reqMsg.code;
rsp.msg = reqMsg.pCont;
taosArrayPush(batchRsp.pRsps, &rsp);
if (NULL == taosArrayPush(batchRsp.pRsps, &rsp)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
rspSize = tSerializeSBatchRsp(NULL, 0, &batchRsp);
if (rspSize < 0) {
qError("tSerializeSBatchRsp failed");
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
pRsp = rpcMallocCont(rspSize);
if (pRsp == NULL) {
qError("rpcMallocCont %d failed", rspSize);
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
if (tSerializeSBatchRsp(pRsp, rspSize, &batchRsp) < 0) {
qError("tSerializeSBatchRsp %d failed", rspSize);
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -501,7 +541,12 @@ int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) {
}
int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnode, uid, 1);
if (NULL == pCur) {
qError("vnode get all table list failed");
return TSDB_CODE_FAILED;
}
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
@ -510,11 +555,15 @@ int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list) {
}
STableKeyInfo info = {uid = id};
taosArrayPush(list, &info);
if (NULL == taosArrayPush(list, &info)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
_exit:
metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg) {
@ -522,8 +571,13 @@ int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
}
int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SVnode *pVnodeObj = pVnode;
SMCtbCursor *pCur = metaOpenCtbCursor(pVnodeObj, suid, 1);
if (NULL == pCur) {
qError("vnode get all table list failed");
return TSDB_CODE_FAILED;
}
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
@ -531,14 +585,20 @@ int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list) {
break;
}
taosArrayPush(list, &id);
if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
_exit:
metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
int32_t code = TSDB_CODE_SUCCESS;
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
if (!pCur) {
return TSDB_CODE_FAILED;
@ -550,15 +610,21 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) {
break;
}
taosArrayPush(list, &id);
if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
_exit:
metaCloseStbCursor(pCur);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1),
void *arg) {
int32_t code = TSDB_CODE_SUCCESS;
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid);
if (!pCur) {
return TSDB_CODE_FAILED;
@ -574,11 +640,16 @@ int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bo
continue;
}
taosArrayPush(list, &id);
if (NULL == taosArrayPush(list, &id)) {
qError("taosArrayPush failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
_exit:
metaCloseStbCursor(pCur);
return TSDB_CODE_SUCCESS;
return code;
}
int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
@ -700,16 +771,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
*num = 0;
int64_t arrSize = taosArrayGetSize(suidList);
int32_t code = TSDB_CODE_SUCCESS;
for (int64_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
int64_t ctbNum = 0;
int32_t numOfCols = 0;
metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols);
code = metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols);
if (TSDB_CODE_SUCCESS != code) {
goto _exit;
}
*num += ctbNum * (numOfCols - 1);
}
_exit:
taosArrayDestroy(suidList);
return TSDB_CODE_SUCCESS;
}
@ -728,7 +803,11 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
}
int64_t ctbNum = 0;
vnodeGetCtbNum(pVnode, id, &ctbNum);
int32_t code = vnodeGetCtbNum(pVnode, id, &ctbNum);
if (TSDB_CODE_SUCCESS != code) {
metaCloseStbCursor(pCur);
return code;
}
*num += ctbNum;
}
@ -771,14 +850,17 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
rsp.vgId = req.vgId;
rsp.streamId = req.streamId;
rspLen = tSerializeStreamProgressRsp(0, 0, &rsp);
if (rspLen < 0) {
code = terrno;
goto _OVER;
}
if (direct) {
buf = rpcMallocCont(rspLen);
} else {
buf = taosMemoryCalloc(1, rspLen);
}
if (!buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
}
@ -787,7 +869,11 @@ int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished);
}
if (code == TSDB_CODE_SUCCESS) {
tSerializeStreamProgressRsp(buf, rspLen, &rsp);
rspLen = tSerializeStreamProgressRsp(buf, rspLen, &rsp);
if (rspLen < 0) {
code = terrno;
goto _OVER;
}
rpcMsg.pCont = buf;
buf = NULL;
rpcMsg.contLen = rspLen;

290
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -17,20 +17,19 @@
#include "vnd.h"
static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) {
int32_t code = -1;
int32_t code = 0;
STsdbFSetPartList *pList = tsdbFSetPartListCreate();
if (pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _out;
}
if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
}
if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) {
goto _out;
}
code = 0;
code = tDeserializeTsdbFSetPartList(buf, bufLen, pList);
if (code) goto _out;
code = tsdbFSetPartListToRangeDiff(pList, ppRanges);
if (code) goto _out;
_out:
tsdbFSetPartListDestroy(&pList);
return code;
@ -88,14 +87,15 @@ static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader,
}
static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
int32_t code = 0;
SVnode *pVnode = pReader->pVnode;
int32_t code = -1;
if (pParam->data) {
// decode
SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
code = TSDB_CODE_INVALID_DATA_FMT;
terrno = code;
goto _out;
}
@ -116,21 +116,25 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
}
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
if (code) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out;
}
} break;
case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
if (code) {
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out;
}
} break;
default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
code = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
}
}
@ -147,7 +151,7 @@ static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotP
vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode),
(pReader->tsdbDone ? "raw" : "normal"));
}
code = 0;
_out:
return code;
}
@ -160,42 +164,43 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
if (pReader == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pReader->pVnode = pVnode;
pReader->sver = sver;
pReader->ever = ever;
// snapshot info
if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) {
goto _err;
}
code = vnodeSnapReaderDealWithSnapInfo(pReader, pParam);
if (code) goto _exit;
// open tsdb snapshot raw reader
if (!pReader->tsdbRAWDone) {
ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pVnode->pTsdb, ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err;
if (code) goto _exit;
}
// check snapshot ever
SSnapshot snapshot = {0};
vnodeGetSnapshot(pVnode, &snapshot);
code = vnodeGetSnapshot(pVnode, &snapshot);
if (code) goto _exit;
if (ever != snapshot.lastApplyIndex) {
vError("vgId:%d, abort reader open due to vnode snapshot changed. ever:%" PRId64 ", commit ver:%" PRId64,
TD_VID(pVnode), ever, snapshot.lastApplyIndex);
code = TSDB_CODE_SYN_INTERNAL_ERROR;
goto _err;
goto _exit;
}
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = pReader;
return code;
_err:
_exit:
if (code) {
vError("vgId:%d, vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
*ppReader = NULL;
} else {
vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
*ppReader = pReader;
}
return code;
}
@ -245,6 +250,7 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
int32_t code = 0;
int32_t lino;
SVnode *pVnode = pReader->pVnode;
int32_t vgId = TD_VID(pReader->pVnode);
@ -261,31 +267,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ);
if (NULL == pFile) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
int64_t size;
if (taosFStatFile(pFile, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG;
((SSnapDataHdr *)(*ppData))->size = size + 1;
((SSnapDataHdr *)(*ppData))->data[size] = '\0';
if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(*ppData);
taosCloseFile(&pFile);
goto _err;
TSDB_CHECK_CODE(code = TAOS_SYSTEM_ERROR(errno), lino, _exit);
}
taosCloseFile(&pFile);
@ -299,20 +302,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
// open reader if not
if (pReader->pMetaReader == NULL) {
code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = metaSnapRead(pReader->pMetaReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->metaDone = 1;
code = metaSnapReaderClose(&pReader->pMetaReader);
if (code) goto _err;
}
metaSnapReaderClose(&pReader->pMetaReader);
}
}
@ -322,20 +322,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbReader == NULL) {
code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges,
&pReader->pTsdbReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapRead(pReader->pTsdbReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->tsdbDone = 1;
code = tsdbSnapReaderClose(&pReader->pTsdbReader);
if (code) goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -344,20 +341,17 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (pReader->pTsdbRAWReader == NULL) {
ASSERT(pReader->sver == 0);
code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->tsdbRAWDone = 1;
code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
if (code) goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -365,59 +359,53 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
vInfo("vgId:%d tq transform start", vgId);
if (!pReader->tqHandleDone) {
if (pReader->pTqSnapReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE, &pReader->pTqSnapReader);
if (code < 0) goto _err;
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_HANDLE,
&pReader->pTqSnapReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapRead(pReader->pTqSnapReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->tqHandleDone = 1;
code = tqSnapReaderClose(&pReader->pTqSnapReader);
if (code) goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
}
if (!pReader->tqCheckInfoDone) {
if (pReader->pTqCheckInfoReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO, &pReader->pTqCheckInfoReader);
if (code < 0) goto _err;
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_CHECKINFO,
&pReader->pTqCheckInfoReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapRead(pReader->pTqCheckInfoReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->tqCheckInfoDone = 1;
code = tqSnapReaderClose(&pReader->pTqCheckInfoReader);
if (code) goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
}
if (!pReader->tqOffsetDone) {
if (pReader->pTqOffsetReader == NULL) {
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET, &pReader->pTqOffsetReader);
if (code < 0) goto _err;
code = tqSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->ever, SNAP_DATA_TQ_OFFSET,
&pReader->pTqOffsetReader);
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapRead(pReader->pTqOffsetReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->tqOffsetDone = 1;
code = tqSnapReaderClose(&pReader->pTqOffsetReader);
if (code) goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -426,30 +414,21 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (!pReader->streamTaskDone) {
if (pReader->pStreamTaskReader == NULL) {
code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader);
if (code) {
vError("vgId:%d open streamtask snapshot reader failed, code:%s", vgId, tstrerror(code));
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
code = streamTaskSnapRead(pReader->pStreamTaskReader, ppData);
if (code) {
vError("vgId:%d error happens during read data from streatask snapshot, code:%s", vgId, tstrerror(code));
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
vInfo("vgId:%d no streamTask snapshot", vgId);
goto _exit;
} else {
pReader->streamTaskDone = 1;
code = streamTaskSnapReaderClose(pReader->pStreamTaskReader);
if (code) {
goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
pReader->pStreamTaskReader = NULL;
}
}
}
if (!pReader->streamStateDone) {
if (pReader->pStreamStateReader == NULL) {
code =
@ -457,43 +436,37 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
if (code) {
pReader->streamStateDone = 1;
pReader->pStreamStateReader = NULL;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = streamStateSnapRead(pReader->pStreamStateReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->streamStateDone = 1;
code = streamStateSnapReaderClose(pReader->pStreamStateReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
pReader->pStreamStateReader = NULL;
}
}
}
// RSMA ==============
if (VND_IS_RSMA(pReader->pVnode) && !pReader->rsmaDone) {
// open if not
if (pReader->pRsmaReader == NULL) {
code = rsmaSnapReaderOpen(pReader->pVnode->pSma, pReader->sver, pReader->ever, &pReader->pRsmaReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = rsmaSnapRead(pReader->pRsmaReader, ppData);
if (code) {
goto _err;
} else {
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
pReader->rsmaDone = 1;
code = rsmaSnapReaderClose(&pReader->pRsmaReader);
if (code) goto _err;
}
TSDB_CHECK_CODE(code, lino, _exit);
}
}
@ -501,6 +474,9 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
*nData = 0;
_exit:
if (code) {
vError("vgId:%d, vnode snapshot read failed at %s:%d since %s", vgId, __FILE__, lino, tstrerror(code));
} else {
if (*ppData) {
SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData);
@ -512,10 +488,7 @@ _exit:
} else {
vInfo("vgId:%d, vnode snapshot read data end, index:%" PRId64, vgId, pReader->index);
}
return code;
_err:
vError("vgId:%d, vnode snapshot read failed since %s", vgId, tstrerror(code));
}
return code;
}
@ -563,13 +536,13 @@ TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t
static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
SVnode *pVnode = pWriter->pVnode;
int32_t code = -1;
int32_t code = 0;
int32_t lino;
if (pParam->data) {
SSyncTLV *datHead = (void *)pParam->data;
if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
terrno = TSDB_CODE_INVALID_DATA_FMT;
goto _out;
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
}
STsdbRepOpts tsdbOpts = {0};
@ -589,30 +562,30 @@ static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotP
ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
if (ppRanges == NULL) {
vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
goto _out;
}
if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
goto _out;
TSDB_CHECK_CODE(code = terrno, lino, _exit);
}
code = vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges);
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_RAW: {
if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
goto _out;
}
code = tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts);
TSDB_CHECK_CODE(code, lino, _exit);
} break;
default:
vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
goto _out;
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_DATA_FMT, lino, _exit);
goto _exit;
}
}
vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
}
code = 0;
_out:
_exit:
if (code) {
vError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, __LINE__, tstrerror(code));
}
return code;
}
@ -634,6 +607,7 @@ static int32_t vnodeEnableBgTask(SVnode *pVnode) {
int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) {
int32_t code = 0;
int32_t lino;
SVSnapWriter *pWriter = NULL;
int64_t sver = pParam->start;
int64_t ever = pParam->end;
@ -644,8 +618,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
// alloc
pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
}
pWriter->pVnode = pVnode;
pWriter->sver = sver;
@ -655,19 +628,19 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
pWriter->commitID = ++pVnode->state.commitID;
// snapshot info
if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) {
goto _err;
}
code = vnodeSnapWriterDealWithSnapInfo(pWriter, pParam);
TSDB_CHECK_CODE(code, lino, _exit);
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
return code;
_err:
_exit:
if (code) {
vError("vgId:%d, vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
if (pWriter) taosMemoryFreeClear(pWriter);
*ppWriter = NULL;
} else {
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID);
*ppWriter = pWriter;
}
return code;
}
@ -785,14 +758,13 @@ _exit:
static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
int32_t lino;
SVnode *pVnode = pWriter->pVnode;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
// decode info
if (vnodeDecodeInfo(pHdr->data, &pWriter->info) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
code = vnodeDecodeInfo(pHdr->data, &pWriter->info);
TSDB_CHECK_CODE(code, lino, _exit);
// change some value
pWriter->info.state.commitID = pWriter->commitID;
@ -805,10 +777,8 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_
pWriter->info.config = pVnode->config;
pWriter->info.config.vndStats = vndStats;
vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId);
if (vnodeSaveInfo(dir, &pWriter->info) < 0) {
code = terrno;
goto _exit;
}
code = vnodeSaveInfo(dir, &pWriter->info);
TSDB_CHECK_CODE(code, lino, _exit);
_exit:
return code;
@ -816,6 +786,7 @@ _exit:
int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
int32_t lino;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
SVnode *pVnode = pWriter->pVnode;
@ -824,7 +795,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pHdr->index != pWriter->index + 1) {
vError("vgId:%d, unexpected vnode snapshot msg. index:%" PRId64 ", expected index:%" PRId64, TD_VID(pVnode),
pHdr->index, pWriter->index + 1);
return -1;
TSDB_CHECK_CODE(code = TSDB_CODE_INVALID_MSG, lino, _exit);
}
pWriter->index = pHdr->index;
@ -835,17 +806,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
switch (pHdr->type) {
case SNAP_DATA_CFG: {
code = vnodeSnapWriteInfo(pWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_META: {
// meta
if (pWriter->pMetaSnapWriter == NULL) {
code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TSDB:
case SNAP_DATA_DEL: {
@ -853,69 +824,69 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pTsdbSnapWriter == NULL) {
code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges,
&pWriter->pTsdbSnapWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_RAW: {
// tsdb
if (pWriter->pTsdbSnapRAWWriter == NULL) {
ASSERT(pWriter->sver == 0);
code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TQ_HANDLE: {
// tq handle
if (pWriter->pTqSnapHandleWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapHandleWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapHandleWrite(pWriter->pTqSnapHandleWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TQ_CHECKINFO: {
// tq checkinfo
if (pWriter->pTqSnapCheckInfoWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapCheckInfoWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapCheckInfoWrite(pWriter->pTqSnapCheckInfoWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_TQ_OFFSET: {
// tq offset
if (pWriter->pTqSnapOffsetWriter == NULL) {
code = tqSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pTqSnapOffsetWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = tqSnapOffsetWrite(pWriter->pTqSnapOffsetWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_STREAM_TASK:
case SNAP_DATA_STREAM_TASK_CHECKPOINT: {
if (pWriter->pStreamTaskWriter == NULL) {
code = streamTaskSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamTaskWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_STREAM_STATE_BACKEND: {
if (pWriter->pStreamStateWriter == NULL) {
code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = streamStateSnapWrite(pWriter->pStreamStateWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
case SNAP_DATA_RSMA1:
@ -925,20 +896,19 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
if (pWriter->pRsmaSnapWriter == NULL) {
code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges,
&pWriter->pRsmaSnapWriter);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
code = rsmaSnapWrite(pWriter->pRsmaSnapWriter, pData, nData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
} break;
default:
break;
}
_exit:
return code;
_err:
if (code) {
vError("vgId:%d, vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode),
tstrerror(code), pHdr->index, pHdr->type, nData);
}
return code;
}

30
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -387,7 +387,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return -1;
return TSDB_CODE_INVALID_PARA;
}
int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg);
@ -400,13 +400,13 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
if (pMsg == NULL || pMsg->pCont == NULL) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
if (msgcb == NULL || msgcb->putToQueueFp == NULL) {
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
return -1;
return TSDB_CODE_INVALID_PARA;
}
int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
@ -485,8 +485,7 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta
static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
return code;
return vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
}
static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
@ -496,8 +495,7 @@ static void vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
SVnode *pVnode = pFsm->data;
int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
return code;
return vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
}
static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
@ -514,8 +512,7 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
}
} while (true);
int32_t code = vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
return code;
return vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
}
static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
@ -662,12 +659,16 @@ static int32_t vnodeApplyQueueItems(const SSyncFSM *pFsm) {
int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE);
return itemSize;
} else {
return -1;
return TSDB_CODE_INVALID_PARA;
}
}
static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
if (pFsm == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitMsg;
pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex;
@ -724,7 +725,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
pVnode->sync = syncOpen(&syncInfo, vnodeVersion);
if (pVnode->sync <= 0) {
vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
return -1;
return terrno;
}
return 0;
@ -732,9 +733,10 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path, int32_t vnodeVersion) {
int32_t vnodeSyncStart(SVnode *pVnode) {
vInfo("vgId:%d, start sync", pVnode->config.vgId);
if (syncStart(pVnode->sync) < 0) {
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, terrstr());
return -1;
int32_t code = syncStart(pVnode->sync);
if (code) {
vError("vgId:%d, failed to start sync subsystem since %s", pVnode->config.vgId, tstrerror(code));
return code;
}
return 0;
}

9
source/libs/command/inc/commandInt.h
View File

@ -176,7 +176,14 @@ typedef struct SExplainCtx {
#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
#define EXPLAIN_MERGE_MODE_STRING(_mode) ((_mode) == MERGE_TYPE_SORT ? "sort" : ((_mode) == MERGE_TYPE_NON_SORT ? "merge" : "column"))
#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u)))
#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p, _r) \
do { \
if ((_u) == 'n' || (_u) == 'y') { \
_r = (_t); \
} else { \
code = convertTimeFromPrecisionToUnit(_t, _p, _u, &_r); \
} \
} while(0)
#define EXPLAIN_ROW_NEW(level, ...) \
do { \

Some files were not shown because too many files have changed in this diff Show More